LLVM 19.0.0git
X86ISelLowering.h
Go to the documentation of this file.
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
15#define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
16
19
20namespace llvm {
21 class X86Subtarget;
22 class X86TargetMachine;
23
24 namespace X86ISD {
25 // X86 Specific DAG Nodes
26 enum NodeType : unsigned {
27 // Start the numbering where the builtin ops leave off.
29
30 /// Bit scan forward.
32 /// Bit scan reverse.
34
35 /// X86 funnel/double shift i16 instructions. These correspond to
36 /// X86::SHLDW and X86::SHRDW instructions which have different amt
37 /// modulo rules to generic funnel shifts.
38 /// NOTE: The operand order matches ISD::FSHL/FSHR not SHLD/SHRD.
41
42 /// Bitwise logical AND of floating point values. This corresponds
43 /// to X86::ANDPS or X86::ANDPD.
45
46 /// Bitwise logical OR of floating point values. This corresponds
47 /// to X86::ORPS or X86::ORPD.
49
50 /// Bitwise logical XOR of floating point values. This corresponds
51 /// to X86::XORPS or X86::XORPD.
53
54 /// Bitwise logical ANDNOT of floating point values. This
55 /// corresponds to X86::ANDNPS or X86::ANDNPD.
57
58 /// These operations represent an abstract X86 call
59 /// instruction, which includes a bunch of information. In particular the
60 /// operands of these node are:
61 ///
62 /// #0 - The incoming token chain
63 /// #1 - The callee
64 /// #2 - The number of arg bytes the caller pushes on the stack.
65 /// #3 - The number of arg bytes the callee pops off the stack.
66 /// #4 - The value to pass in AL/AX/EAX (optional)
67 /// #5 - The value to pass in DL/DX/EDX (optional)
68 ///
69 /// The result values of these nodes are:
70 ///
71 /// #0 - The outgoing token chain
72 /// #1 - The first register result value (optional)
73 /// #2 - The second register result value (optional)
74 ///
76
77 /// Same as call except it adds the NoTrack prefix.
79
80 // Pseudo for a OBJC call that gets emitted together with a special
81 // marker instruction.
83
84 /// X86 compare and logical compare instructions.
89
90 /// X86 bit-test instructions.
92
93 /// X86 SetCC. Operand 0 is condition code, and operand 1 is the EFLAGS
94 /// operand, usually produced by a CMP instruction.
96
97 /// X86 Select
99
100 // Same as SETCC except it's materialized with a sbb and the value is all
101 // one's or all zero's.
102 SETCC_CARRY, // R = carry_bit ? ~0 : 0
103
104 /// X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
105 /// Operands are two FP values to compare; result is a mask of
106 /// 0s or 1s. Generally DTRT for C/C++ with NaNs.
108
109 /// X86 FP SETCC, similar to above, but with output as an i1 mask and
110 /// and a version with SAE.
113
114 /// X86 conditional moves. Operand 0 and operand 1 are the two values
115 /// to select from. Operand 2 is the condition code, and operand 3 is the
116 /// flag operand produced by a CMP or TEST instruction.
118
119 /// X86 conditional branches. Operand 0 is the chain operand, operand 1
120 /// is the block to branch if condition is true, operand 2 is the
121 /// condition code, and operand 3 is the flag operand produced by a CMP
122 /// or TEST instruction.
124
125 /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and
126 /// operand 1 is the target address.
128
129 /// Return with a glue operand. Operand 0 is the chain operand, operand
130 /// 1 is the number of bytes of stack to pop.
132
133 /// Return from interrupt. Operand 0 is the number of bytes to pop.
135
136 /// Repeat fill, corresponds to X86::REP_STOSx.
138
139 /// Repeat move, corresponds to X86::REP_MOVSx.
141
142 /// On Darwin, this node represents the result of the popl
143 /// at function entry, used for PIC code.
145
146 /// A wrapper node for TargetConstantPool, TargetJumpTable,
147 /// TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress,
148 /// MCSymbol and TargetBlockAddress.
150
151 /// Special wrapper used under X86-64 PIC mode for RIP
152 /// relative displacements.
154
155 /// Copies a 64-bit value from an MMX vector to the low word
156 /// of an XMM vector, with the high word zero filled.
158
159 /// Copies a 64-bit value from the low word of an XMM vector
160 /// to an MMX vector.
162
163 /// Copies a 32-bit value from the low word of a MMX
164 /// vector to a GPR.
166
167 /// Copies a GPR into the low 32-bit word of a MMX vector
168 /// and zero out the high word.
170
171 /// Extract an 8-bit value from a vector and zero extend it to
172 /// i32, corresponds to X86::PEXTRB.
174
175 /// Extract a 16-bit value from a vector and zero extend it to
176 /// i32, corresponds to X86::PEXTRW.
178
179 /// Insert any element of a 4 x float vector into any element
180 /// of a destination 4 x floatvector.
182
183 /// Insert the lower 8-bits of a 32-bit value to a vector,
184 /// corresponds to X86::PINSRB.
186
187 /// Insert the lower 16-bits of a 32-bit value to a vector,
188 /// corresponds to X86::PINSRW.
190
191 /// Shuffle 16 8-bit values within a vector.
193
194 /// Compute Sum of Absolute Differences.
196 /// Compute Double Block Packed Sum-Absolute-Differences
198
199 /// Bitwise Logical AND NOT of Packed FP values.
201
202 /// Blend where the selector is an immediate.
204
205 /// Dynamic (non-constant condition) vector blend where only the sign bits
206 /// of the condition elements are used. This is used to enforce that the
207 /// condition mask is not valid for generic VSELECT optimizations. This
208 /// is also used to implement the intrinsics.
209 /// Operands are in VSELECT order: MASK, TRUE, FALSE
211
212 /// Combined add and sub on an FP vector.
214
215 // FP vector ops with rounding mode.
235
236 // FP vector get exponent.
241 // Extract Normalized Mantissas.
246 // FP Scale.
251
252 /// Integer horizontal add/sub.
255
256 /// Floating point horizontal add/sub.
259
260 // Detect Conflicts Within a Vector
262
263 /// Floating point max and min.
266
267 /// Commutative FMIN and FMAX.
270
271 /// Scalar intrinsic floating point max and min.
274
275 /// Floating point reciprocal-sqrt and reciprocal approximation.
276 /// Note that these typically require refinement
277 /// in order to obtain suitable precision.
280
281 // AVX-512 reciprocal approximations with a little more precision.
286
287 // Thread Local Storage.
289
290 // Thread Local Storage. A call to get the start address
291 // of the TLS block for the current module.
293
294 // Thread Local Storage. When calling to an OS provided
295 // thunk at the address from an earlier relocation.
297
298 // Thread Local Storage. A descriptor containing pointer to
299 // code and to argument to get the TLS offset for the symbol.
301
302 // Exception Handling helpers.
304
305 // SjLj exception handling setjmp.
307
308 // SjLj exception handling longjmp.
310
311 // SjLj exception handling dispatch.
313
314 /// Tail call return. See X86TargetLowering::LowerCall for
315 /// the list of operands.
317
318 // Vector move to low scalar and zero higher vector elements.
320
321 // Vector integer truncate.
323 // Vector integer truncate with unsigned/signed saturation.
326
327 // Masked version of the above. Used when less than a 128-bit result is
328 // produced since the mask only applies to the lower elements and can't
329 // be represented by a select.
330 // SRC, PASSTHRU, MASK
334
335 // Vector FP extend.
340
341 // Vector FP round.
346
347 // Masked version of above. Used for v2f64->v4f32.
348 // SRC, PASSTHRU, MASK
350
351 // 128-bit vector logical left / right shift
354
355 // Vector shift elements
359
360 // Vector variable shift
364
365 // Vector shift elements by immediate
369
370 // Shifts of mask registers.
373
374 // Bit rotate by immediate
377
378 // Vector packed double/float comparison.
380
381 // Vector integer comparisons.
384
385 // v8i16 Horizontal minimum and position.
387
389
390 /// Vector comparison generating mask bits for fp and
391 /// integer signed and unsigned data types.
393 // Vector mask comparison generating mask bits for FP values.
395 // Vector mask comparison with SAE for FP values.
397
398 // Arithmetic operations with FLAGS results.
408
409 // Bit field extract.
412
413 // Zero High Bits Starting with Specified Bit Position.
415
416 // Parallel extract and deposit.
419
420 // X86-specific multiply by immediate.
422
423 // Vector sign bit extraction.
425
426 // Vector bitwise comparisons.
428
429 // Vector packed fp sign bitwise comparisons.
431
432 // OR/AND test for masks.
435
436 // ADD for masks.
438
439 // Several flavors of instructions with vector shuffle behaviors.
440 // Saturated signed/unnsigned packing.
443 // Intra-lane alignr.
445 // AVX512 inter-lane alignr.
451 // VBMI2 Concat & Shift.
456 // Shuffle Packed Values at 128-bit granularity.
472
473 // Variable Permute (VPERM).
474 // Res = VPERMV MaskV, V0
476
477 // 3-op Variable Permute (VPERMT2).
478 // Res = VPERMV3 V0, MaskV, V1
480
481 // Bitwise ternary logic.
483 // Fix Up Special Packed Float32/64 values.
488 // Range Restriction Calculation For Packed Pairs of Float32/64 values.
493 // Reduce - Perform Reduction Transformation on scalar\packed FP.
498 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
499 // Also used by the legacy (V)ROUND intrinsics where we mask out the
500 // scaling part of the immediate.
505 // Tests Types Of a FP Values for packed types.
507 // Tests Types Of a FP Values for scalar types.
509
510 // Broadcast (splat) scalar or element 0 of a vector. If the operand is
511 // a vector, this node may change the vector length as part of the splat.
513 // Broadcast mask to vector.
515
516 /// SSE4A Extraction and Insertion.
519
520 // XOP arithmetic/logical shifts.
523 // XOP signed/unsigned integer comparisons.
526 // XOP packed permute bytes.
528 // XOP two source permutation.
530
531 // Vector multiply packed unsigned doubleword integers.
533 // Vector multiply packed signed doubleword integers.
535 // Vector Multiply Packed UnsignedIntegers with Round and Scale.
537
538 // Multiply and Add Packed Integers.
541
542 // AVX512IFMA multiply and add.
543 // NOTE: These are different than the instruction and perform
544 // op0 x op1 + op2.
547
548 // VNNI
553
554 // FMA nodes.
555 // We use the target independent ISD::FMA for the non-inverted case.
561
562 // FMA with rounding mode.
569
570 // AVX512-FP16 complex addition and multiplication.
575
580
585
590
597
598 // Compress and expand.
601
602 // Bits shuffle
604
605 // Convert Unsigned/Integer to Floating-Point Value with rounding mode.
612
613 // Vector float/double to signed/unsigned integer.
618 // Scalar float/double to signed/unsigned integer.
623
624 // Vector float/double to signed/unsigned integer with truncation.
629 // Scalar float/double to signed/unsigned integer with truncation.
634
635 // Vector signed/unsigned integer to float/double.
638
639 // Masked versions of above. Used for v2f64->v4f32.
640 // SRC, PASSTHRU, MASK
647
648 // Vector float to bfloat16.
649 // Convert TWO packed single data to one packed BF16 data
651 // Convert packed single data to packed BF16 data
653 // Masked version of above.
654 // SRC, PASSTHRU, MASK
656
657 // Dot product of BF16 pairs to accumulated into
658 // packed single precision.
660
661 // A stack checking function call. On Windows it's _chkstk call.
663
664 // For allocating variable amounts of stack space when using
665 // segmented stacks. Check if the current stacklet has enough space, and
666 // falls back to heap allocation if not.
668
669 // For allocating stack space when using stack clash protector.
670 // Allocation is performed by block, and each block is probed.
672
673 // Memory barriers.
675
676 // Get a random integer and indicate whether it is valid in CF.
678
679 // Get a NIST SP800-90B & C compliant random integer and
680 // indicate whether it is valid in CF.
682
683 // Protection keys
684 // RDPKRU - Operand 0 is chain. Operand 1 is value for ECX.
685 // WRPKRU - Operand 0 is chain. Operand 1 is value for EDX. Operand 2 is
686 // value for ECX.
689
690 // SSE42 string comparisons.
691 // These nodes produce 3 results, index, mask, and flags. X86ISelDAGToDAG
692 // will emit one or two instructions based on which results are used. If
693 // flags and index/mask this allows us to use a single instruction since
694 // we won't have to pick and opcode for flags. Instead we can rely on the
695 // DAG to CSE everything and decide at isel.
698
699 // Test if in transactional execution.
701
702 // Conversions between float and half-float.
707
708 // Masked version of above.
709 // SRC, RND, PASSTHRU, MASK
712
713 // Galois Field Arithmetic Instructions
717
718 // LWP insert record.
720
721 // User level wait
724
725 // Enqueue Stores Instructions
728
729 // For avx512-vp2intersect
731
732 // User level interrupts - testui
734
735 // Perform an FP80 add after changing precision control in FPCW.
737
738 // Conditional compare instructions
741
742 /// X86 strict FP compare instructions.
745
746 // Vector packed double/float comparison.
748
749 /// Vector comparison generating mask bits for fp and
750 /// integer signed and unsigned data types.
752
753 // Vector float/double to signed/unsigned integer with truncation.
756
757 // Vector FP extend.
759
760 // Vector FP round.
762
763 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
764 // Also used by the legacy (V)ROUND intrinsics where we mask out the
765 // scaling part of the immediate.
767
768 // Vector signed/unsigned integer to float/double.
771
772 // Strict FMA nodes.
776
777 // Conversions between float and half-float.
780
781 // Perform an FP80 add after changing precision control in FPCW.
783
784 // WARNING: Only add nodes here if they are strict FP nodes. Non-memory and
785 // non-strict FP nodes should be above FIRST_TARGET_STRICTFP_OPCODE.
786
787 // Compare and swap.
792
793 /// LOCK-prefixed arithmetic read-modify-write instructions.
794 /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)
806
807 /// RAO arithmetic instructions.
808 /// OUTCHAIN = AADD(INCHAIN, PTR, RHS)
813
814 // Load, scalar_to_vector, and zero extend.
816
817 // extract_vector_elt, store.
819
820 // scalar broadcast from memory.
822
823 // subvector broadcast from memory.
825
826 // Store FP control word into i16 memory.
828
829 // Load FP control word from i16 memory.
831
832 // Store x87 FPU environment into memory.
834
835 // Load x87 FPU environment from memory.
837
838 /// This instruction implements FP_TO_SINT with the
839 /// integer destination in memory and a FP reg source. This corresponds
840 /// to the X86::FIST*m instructions and the rounding mode change stuff. It
841 /// has two inputs (token chain and address) and two outputs (int value
842 /// and token chain). Memory VT specifies the type to store to.
844
845 /// This instruction implements SINT_TO_FP with the
846 /// integer source in memory and FP reg result. This corresponds to the
847 /// X86::FILD*m instructions. It has two inputs (token chain and address)
848 /// and two outputs (FP value and token chain). The integer source type is
849 /// specified by the memory VT.
851
852 /// This instruction implements a fp->int store from FP stack
853 /// slots. This corresponds to the fist instruction. It takes a
854 /// chain operand, value to store, address, and glue. The memory VT
855 /// specifies the type to store as.
857
858 /// This instruction implements an extending load to FP stack slots.
859 /// This corresponds to the X86::FLD32m / X86::FLD64m. It takes a chain
860 /// operand, and ptr to load from. The memory VT specifies the type to
861 /// load from.
863
864 /// This instruction implements a truncating store from FP stack
865 /// slots. This corresponds to the X86::FST32m / X86::FST64m. It takes a
866 /// chain operand, value to store, address, and glue. The memory VT
867 /// specifies the type to store as.
869
870 /// These instructions grab the address of the next argument
871 /// from a va_list. (reads and modifies the va_list in memory)
874
875 // Vector truncating store with unsigned/signed saturation
878 // Vector truncating masked store with unsigned/signed saturation
881
882 // X86 specific gather and scatter
885
886 // Key locker nodes that produce flags.
895
896 /// Compare and Add if Condition is Met. Compare value in operand 2 with
897 /// value in memory of operand 1. If condition of operand 4 is met, add
898 /// value operand 3 to m32 and write new value in operand 1. Operand 2 is
899 /// always updated with the original value from operand 1.
901
902 // Save xmm argument registers to the stack, according to %al. An operator
903 // is needed so that this can be expanded with control flow.
905
906 // WARNING: Do not add anything in the end unless you want the node to
907 // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
908 // opcodes will be thought as target memory ops!
909 };
910 } // end namespace X86ISD
911
912 namespace X86 {
913 /// Current rounding mode is represented in bits 11:10 of FPSR. These
914 /// values are same as corresponding constants for rounding mode used
915 /// in glibc.
917 rmToNearest = 0, // FE_TONEAREST
918 rmDownward = 1 << 10, // FE_DOWNWARD
919 rmUpward = 2 << 10, // FE_UPWARD
920 rmTowardZero = 3 << 10, // FE_TOWARDZERO
921 rmMask = 3 << 10 // Bit mask selecting rounding mode
922 };
923 }
924
925 /// Define some predicates that are used for node matching.
926 namespace X86 {
927 /// Returns true if Elt is a constant zero or floating point constant +0.0.
928 bool isZeroNode(SDValue Elt);
929
930 /// Returns true of the given offset can be
931 /// fit into displacement field of the instruction.
933 bool hasSymbolicDisplacement);
934
935 /// Determines whether the callee is required to pop its
936 /// own arguments. Callee pop is necessary to support tail calls.
937 bool isCalleePop(CallingConv::ID CallingConv,
938 bool is64Bit, bool IsVarArg, bool GuaranteeTCO);
939
940 /// If Op is a constant whose elements are all the same constant or
941 /// undefined, return true and return the constant value in \p SplatVal.
942 /// If we have undef bits that don't cover an entire element, we treat these
943 /// as zero if AllowPartialUndefs is set, else we fail and return false.
944 bool isConstantSplat(SDValue Op, APInt &SplatVal,
945 bool AllowPartialUndefs = true);
946
947 /// Check if Op is a load operation that could be folded into some other x86
948 /// instruction as a memory operand. Example: vpaddd (%rdi), %xmm0, %xmm0.
949 bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
950 bool AssumeSingleUse = false);
951
952 /// Check if Op is a load operation that could be folded into a vector splat
953 /// instruction as a memory operand. Example: vbroadcastss 16(%rdi), %xmm2.
954 bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
955 const X86Subtarget &Subtarget,
956 bool AssumeSingleUse = false);
957
958 /// Check if Op is a value that could be used to fold a store into some
959 /// other x86 instruction as a memory operand. Ex: pextrb $0, %xmm0, (%rdi).
960 bool mayFoldIntoStore(SDValue Op);
961
962 /// Check if Op is an operation that could be folded into a zero extend x86
963 /// instruction.
964 bool mayFoldIntoZeroExtend(SDValue Op);
965
966 /// True if the target supports the extended frame for async Swift
967 /// functions.
968 bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
969 const MachineFunction &MF);
970 } // end namespace X86
971
972 //===--------------------------------------------------------------------===//
973 // X86 Implementation of the TargetLowering interface
974 class X86TargetLowering final : public TargetLowering {
975 public:
976 explicit X86TargetLowering(const X86TargetMachine &TM,
977 const X86Subtarget &STI);
978
979 unsigned getJumpTableEncoding() const override;
980 bool useSoftFloat() const override;
981
982 void markLibCallAttributes(MachineFunction *MF, unsigned CC,
983 ArgListTy &Args) const override;
984
985 MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {
986 return MVT::i8;
987 }
988
989 const MCExpr *
991 const MachineBasicBlock *MBB, unsigned uid,
992 MCContext &Ctx) const override;
993
994 /// Returns relocation base for the given PIC jumptable.
996 SelectionDAG &DAG) const override;
997 const MCExpr *
999 unsigned JTI, MCContext &Ctx) const override;
1000
1001 /// Return the desired alignment for ByVal aggregate
1002 /// function arguments in the caller parameter area. For X86, aggregates
1003 /// that contains are placed at 16-byte boundaries while the rest are at
1004 /// 4-byte boundaries.
1006 const DataLayout &DL) const override;
1007
1009 const AttributeList &FuncAttributes) const override;
1010
1011 /// Returns true if it's safe to use load / store of the
1012 /// specified type to expand memcpy / memset inline. This is mostly true
1013 /// for all types except for some special cases. For example, on X86
1014 /// targets without SSE2 f64 load / store are done with fldl / fstpl which
1015 /// also does type conversion. Note the specified type doesn't have to be
1016 /// legal as the hook is used before type legalization.
1017 bool isSafeMemOpType(MVT VT) const override;
1018
1019 bool isMemoryAccessFast(EVT VT, Align Alignment) const;
1020
1021 /// Returns true if the target allows unaligned memory accesses of the
1022 /// specified type. Returns whether it is "fast" in the last argument.
1023 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
1025 unsigned *Fast) const override;
1026
1027 /// This function returns true if the memory access is aligned or if the
1028 /// target allows this specific unaligned memory access. If the access is
1029 /// allowed, the optional final parameter returns a relative speed of the
1030 /// access (as defined by the target).
1031 bool allowsMemoryAccess(
1032 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1033 Align Alignment,
1035 unsigned *Fast = nullptr) const override;
1036
1038 const MachineMemOperand &MMO,
1039 unsigned *Fast) const {
1040 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
1041 MMO.getAlign(), MMO.getFlags(), Fast);
1042 }
1043
1044 /// Provide custom lowering hooks for some operations.
1045 ///
1046 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
1047
1048 /// Replace the results of node with an illegal result
1049 /// type with new values built out of custom code.
1050 ///
1052 SelectionDAG &DAG) const override;
1053
1054 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
1055
1056 bool preferABDSToABSWithNSW(EVT VT) const override;
1057
1058 bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
1059 EVT ExtVT) const override;
1060
1062 EVT VT) const override;
1063
1064 /// Return true if the target has native support for
1065 /// the specified value type and it is 'desirable' to use the type for the
1066 /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
1067 /// instruction encodings are longer and some i16 instructions are slow.
1068 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
1069
1070 /// Return true if the target has native support for the
1071 /// specified value type and it is 'desirable' to use the type. e.g. On x86
1072 /// i16 is legal, but undesirable since i16 instruction encodings are longer
1073 /// and some i16 instructions are slow.
1074 bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
1075
1076 /// Return prefered fold type, Abs if this is a vector, AddAnd if its an
1077 /// integer, None otherwise.
1080 const SDNode *SETCC0,
1081 const SDNode *SETCC1) const override;
1082
1083 /// Return the newly negated expression if the cost is not expensive and
1084 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
1085 /// do the negation.
1087 bool LegalOperations, bool ForCodeSize,
1089 unsigned Depth) const override;
1090
1093 MachineBasicBlock *MBB) const override;
1094
1095 /// This method returns the name of a target specific DAG node.
1096 const char *getTargetNodeName(unsigned Opcode) const override;
1097
1098 /// Do not merge vector stores after legalization because that may conflict
1099 /// with x86-specific store splitting optimizations.
1100 bool mergeStoresAfterLegalization(EVT MemVT) const override {
1101 return !MemVT.isVector();
1102 }
1103
1104 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
1105 const MachineFunction &MF) const override;
1106
1107 bool isCheapToSpeculateCttz(Type *Ty) const override;
1108
1109 bool isCheapToSpeculateCtlz(Type *Ty) const override;
1110
1111 bool isCtlzFast() const override;
1112
1113 bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {
1114 // If the pair to store is a mixture of float and int values, we will
1115 // save two bitwise instructions and one float-to-int instruction and
1116 // increase one store instruction. There is potentially a more
1117 // significant benefit because it avoids the float->int domain switch
1118 // for input value. So It is more likely a win.
1119 if ((LTy.isFloatingPoint() && HTy.isInteger()) ||
1120 (LTy.isInteger() && HTy.isFloatingPoint()))
1121 return true;
1122 // If the pair only contains int values, we will save two bitwise
1123 // instructions and increase one store instruction (costing one more
1124 // store buffer). Since the benefit is more blurred so we leave
1125 // such pair out until we get testcase to prove it is a win.
1126 return false;
1127 }
1128
1129 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
1130
1131 bool hasAndNotCompare(SDValue Y) const override;
1132
1133 bool hasAndNot(SDValue Y) const override;
1134
1135 bool hasBitTest(SDValue X, SDValue Y) const override;
1136
1139 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
1140 SelectionDAG &DAG) const override;
1141
1143 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
1144 const APInt &ShiftOrRotateAmt,
1145 const std::optional<APInt> &AndMask) const override;
1146
1147 bool preferScalarizeSplat(SDNode *N) const override;
1148
1149 CondMergingParams
1151 const Value *Rhs) const override;
1152
1154 CombineLevel Level) const override;
1155
1156 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;
1157
1158 bool
1160 unsigned KeptBits) const override {
1161 // For vectors, we don't have a preference..
1162 if (XVT.isVector())
1163 return false;
1164
1165 auto VTIsOk = [](EVT VT) -> bool {
1166 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
1167 VT == MVT::i64;
1168 };
1169
1170 // We are ok with KeptBitsVT being byte/word/dword, what MOVS supports.
1171 // XVT will be larger than KeptBitsVT.
1172 MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
1173 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
1174 }
1175
1178 unsigned ExpansionFactor) const override;
1179
1180 bool shouldSplatInsEltVarIndex(EVT VT) const override;
1181
1182 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {
1183 // Converting to sat variants holds little benefit on X86 as we will just
1184 // need to saturate the value back using fp arithmatic.
1186 }
1187
1188 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
1189 return VT.isScalarInteger();
1190 }
1191
1192 /// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
1193 MVT hasFastEqualityCompare(unsigned NumBits) const override;
1194
1195 /// Return the value type to use for ISD::SETCC.
1197 EVT VT) const override;
1198
1200 const APInt &DemandedElts,
1201 TargetLoweringOpt &TLO) const override;
1202
1203 /// Determine which of the bits specified in Mask are known to be either
1204 /// zero or one and return them in the KnownZero/KnownOne bitsets.
1206 KnownBits &Known,
1207 const APInt &DemandedElts,
1208 const SelectionDAG &DAG,
1209 unsigned Depth = 0) const override;
1210
1211 /// Determine the number of bits in the operation that are sign bits.
1213 const APInt &DemandedElts,
1214 const SelectionDAG &DAG,
1215 unsigned Depth) const override;
1216
1218 const APInt &DemandedElts,
1219 APInt &KnownUndef,
1220 APInt &KnownZero,
1221 TargetLoweringOpt &TLO,
1222 unsigned Depth) const override;
1223
1225 const APInt &DemandedElts,
1226 unsigned MaskIndex,
1227 TargetLoweringOpt &TLO,
1228 unsigned Depth) const;
1229
1231 const APInt &DemandedBits,
1232 const APInt &DemandedElts,
1233 KnownBits &Known,
1234 TargetLoweringOpt &TLO,
1235 unsigned Depth) const override;
1236
1238 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1239 SelectionDAG &DAG, unsigned Depth) const override;
1240
1242 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1243 bool PoisonOnly, unsigned Depth) const override;
1244
1246 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1247 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override;
1248
1249 bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
1250 APInt &UndefElts, const SelectionDAG &DAG,
1251 unsigned Depth) const override;
1252
1254 // Peek through bitcasts/extracts/inserts to see if we have a broadcast
1255 // vector from memory.
1256 while (Op.getOpcode() == ISD::BITCAST ||
1257 Op.getOpcode() == ISD::EXTRACT_SUBVECTOR ||
1258 (Op.getOpcode() == ISD::INSERT_SUBVECTOR &&
1259 Op.getOperand(0).isUndef()))
1260 Op = Op.getOperand(Op.getOpcode() == ISD::INSERT_SUBVECTOR ? 1 : 0);
1261
1262 return Op.getOpcode() == X86ISD::VBROADCAST_LOAD ||
1264 }
1265
1266 const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
1267
1268 SDValue unwrapAddress(SDValue N) const override;
1269
1271
1272 bool ExpandInlineAsm(CallInst *CI) const override;
1273
1274 ConstraintType getConstraintType(StringRef Constraint) const override;
1275
1276 /// Examine constraint string and operand type and determine a weight value.
1277 /// The operand object must already have been set up with the operand type.
1279 getSingleConstraintMatchWeight(AsmOperandInfo &Info,
1280 const char *Constraint) const override;
1281
1282 const char *LowerXConstraint(EVT ConstraintVT) const override;
1283
1284 /// Lower the specified operand into the Ops vector. If it is invalid, don't
1285 /// add anything to Ops. If hasMemory is true it means one of the asm
1286 /// constraint of the inline asm instruction being processed is 'm'.
1288 std::vector<SDValue> &Ops,
1289 SelectionDAG &DAG) const override;
1290
1292 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
1293 if (ConstraintCode == "v")
1295 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
1296 }
1297
1298 /// Handle Lowering flag assembly outputs.
1300 const SDLoc &DL,
1301 const AsmOperandInfo &Constraint,
1302 SelectionDAG &DAG) const override;
1303
1304 /// Given a physical register constraint
1305 /// (e.g. {edx}), return the register number and the register class for the
1306 /// register. This should only be used for C_Register constraints. On
1307 /// error, this returns a register number of 0.
1308 std::pair<unsigned, const TargetRegisterClass *>
1310 StringRef Constraint, MVT VT) const override;
1311
1312 /// Return true if the addressing mode represented
1313 /// by AM is legal for this target, for a load/store of the specified type.
1314 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
1315 Type *Ty, unsigned AS,
1316 Instruction *I = nullptr) const override;
1317
1318 bool addressingModeSupportsTLS(const GlobalValue &GV) const override;
1319
1320 /// Return true if the specified immediate is legal
1321 /// icmp immediate, that is the target has icmp instructions which can
1322 /// compare a register against the immediate without having to materialize
1323 /// the immediate into a register.
1324 bool isLegalICmpImmediate(int64_t Imm) const override;
1325
1326 /// Return true if the specified immediate is legal
1327 /// add immediate, that is the target has add instructions which can
1328 /// add a register and the immediate without having to materialize
1329 /// the immediate into a register.
1330 bool isLegalAddImmediate(int64_t Imm) const override;
1331
1332 bool isLegalStoreImmediate(int64_t Imm) const override;
1333
1334 /// This is used to enable splatted operand transforms for vector shifts
1335 /// and vector funnel shifts.
1336 bool isVectorShiftByScalarCheap(Type *Ty) const override;
1337
1338 /// Add x86-specific opcodes to the default list.
1339 bool isBinOp(unsigned Opcode) const override;
1340
1341 /// Returns true if the opcode is a commutative binary operation.
1342 bool isCommutativeBinOp(unsigned Opcode) const override;
1343
1344 /// Return true if it's free to truncate a value of
1345 /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
1346 /// register EAX to i16 by referencing its sub-register AX.
1347 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
1348 bool isTruncateFree(EVT VT1, EVT VT2) const override;
1349
1350 bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
1351
1352 /// Return true if any actual instruction that defines a
1353 /// value of type Ty1 implicit zero-extends the value to Ty2 in the result
1354 /// register. This does not necessarily include registers defined in
1355 /// unknown ways, such as incoming arguments, or copies from unknown
1356 /// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this
1357 /// does not necessarily apply to truncate instructions. e.g. on x86-64,
1358 /// all instructions that define 32-bit values implicit zero-extend the
1359 /// result out to 64 bits.
1360 bool isZExtFree(Type *Ty1, Type *Ty2) const override;
1361 bool isZExtFree(EVT VT1, EVT VT2) const override;
1362 bool isZExtFree(SDValue Val, EVT VT2) const override;
1363
1365 SmallVectorImpl<Use *> &Ops) const override;
1366 bool shouldConvertPhiType(Type *From, Type *To) const override;
1367
1368 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
1369 /// extend node) is profitable.
1370 bool isVectorLoadExtDesirable(SDValue) const override;
1371
1372 /// Return true if an FMA operation is faster than a pair of fmul and fadd
1373 /// instructions. fmuladd intrinsics will be expanded to FMAs when this
1374 /// method returns true, otherwise fmuladd is expanded to fmul + fadd.
1376 EVT VT) const override;
1377
1378 /// Return true if it's profitable to narrow operations of type SrcVT to
1379 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not
1380 /// from i32 to i16.
1381 bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override;
1382
1383 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
1384 EVT VT) const override;
1385
1386 /// Given an intrinsic, checks if on the target the intrinsic will need to map
1387 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
1388 /// true and stores the intrinsic information into the IntrinsicInfo that was
1389 /// passed to the function.
1390 bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
1391 MachineFunction &MF,
1392 unsigned Intrinsic) const override;
1393
1394 /// Returns true if the target can instruction select the
1395 /// specified FP immediate natively. If false, the legalizer will
1396 /// materialize the FP immediate as a load from a constant pool.
1397 bool isFPImmLegal(const APFloat &Imm, EVT VT,
1398 bool ForCodeSize) const override;
1399
1400 /// Targets can use this to indicate that they only support *some*
1401 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
1402 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to
1403 /// be legal.
1404 bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
1405
1406 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
1407 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
1408 /// constant pool entry.
1409 bool isVectorClearMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
1410
1411 /// Returns true if lowering to a jump table is allowed.
1412 bool areJTsAllowed(const Function *Fn) const override;
1413
1415 EVT ConditionVT) const override;
1416
1417 /// If true, then instruction selection should
1418 /// seek to shrink the FP constant of the specified type to a smaller type
1419 /// in order to save space and / or reduce runtime.
1420 bool ShouldShrinkFPConstant(EVT VT) const override;
1421
1422 /// Return true if we believe it is correct and profitable to reduce the
1423 /// load node to a smaller type.
1425 EVT NewVT) const override;
1426
1427 /// Return true if the specified scalar FP type is computed in an SSE
1428 /// register, not on the X87 floating point stack.
1429 bool isScalarFPTypeInSSEReg(EVT VT) const;
1430
1431 /// Returns true if it is beneficial to convert a load of a constant
1432 /// to just the constant itself.
1434 Type *Ty) const override;
1435
1436 bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override;
1437
1438 bool convertSelectOfConstantsToMath(EVT VT) const override;
1439
1440 bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
1441 SDValue C) const override;
1442
1443 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
1444 /// with this index.
1445 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
1446 unsigned Index) const override;
1447
1448 /// Scalar ops always have equal or better analysis/performance/power than
1449 /// the vector equivalent, so this always makes sense if the scalar op is
1450 /// supported.
1451 bool shouldScalarizeBinop(SDValue) const override;
1452
1453 /// Extract of a scalar FP value from index 0 of a vector is free.
1454 bool isExtractVecEltCheap(EVT VT, unsigned Index) const override {
1455 EVT EltVT = VT.getScalarType();
1456 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
1457 }
1458
1459 /// Overflow nodes should get combined/lowered to optimal instructions
1460 /// (they should allow eliminating explicit compares by getting flags from
1461 /// math ops).
1462 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
1463 bool MathUsed) const override;
1464
1465 bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
1466 unsigned AddrSpace) const override {
1467 // If we can replace more than 2 scalar stores, there will be a reduction
1468 // in instructions even after we add a vector constant load.
1469 return IsZero || NumElem > 2;
1470 }
1471
1472 bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
1473 const SelectionDAG &DAG,
1474 const MachineMemOperand &MMO) const override;
1475
1476 Register getRegisterByName(const char* RegName, LLT VT,
1477 const MachineFunction &MF) const override;
1478
1479 /// If a physical register, this returns the register that receives the
1480 /// exception address on entry to an EH pad.
1481 Register
1482 getExceptionPointerRegister(const Constant *PersonalityFn) const override;
1483
1484 /// If a physical register, this returns the register that receives the
1485 /// exception typeid on entry to a landing pad.
1486 Register
1487 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
1488
1489 bool needsFixedCatchObjects() const override;
1490
1491 /// This method returns a target specific FastISel object,
1492 /// or null if the target does not support "fast" ISel.
1494 const TargetLibraryInfo *libInfo) const override;
1495
1496 /// If the target has a standard location for the stack protector cookie,
1497 /// returns the address of that location. Otherwise, returns nullptr.
1498 Value *getIRStackGuard(IRBuilderBase &IRB) const override;
1499
1500 bool useLoadStackGuardNode() const override;
1501 bool useStackGuardXorFP() const override;
1502 void insertSSPDeclarations(Module &M) const override;
1503 Value *getSDagStackGuard(const Module &M) const override;
1504 Function *getSSPStackGuardCheck(const Module &M) const override;
1506 const SDLoc &DL) const override;
1507
1508
1509 /// Return true if the target stores SafeStack pointer at a fixed offset in
1510 /// some non-standard address space, and populates the address space and
1511 /// offset as appropriate.
1512 Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override;
1513
1514 std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,
1515 SDValue Chain, SDValue Pointer,
1516 MachinePointerInfo PtrInfo,
1517 Align Alignment,
1518 SelectionDAG &DAG) const;
1519
1520 /// Customize the preferred legalization strategy for certain types.
1522
1523 bool softPromoteHalfType() const override { return true; }
1524
1526 EVT VT) const override;
1527
1530 EVT VT) const override;
1531
1533 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1534 unsigned &NumIntermediates, MVT &RegisterVT) const override;
1535
1536 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
1537
1538 bool supportSwiftError() const override;
1539
1540 bool supportKCFIBundles() const override { return true; }
1541
1544 const TargetInstrInfo *TII) const override;
1545
1546 bool hasStackProbeSymbol(const MachineFunction &MF) const override;
1547 bool hasInlineStackProbe(const MachineFunction &MF) const override;
1548 StringRef getStackProbeSymbolName(const MachineFunction &MF) const override;
1549
1550 unsigned getStackProbeSize(const MachineFunction &MF) const;
1551
1552 bool hasVectorBlend() const override { return true; }
1553
1554 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
1555
1557 unsigned OpNo) const override;
1558
1559 /// Lower interleaved load(s) into target specific
1560 /// instructions/intrinsics.
1563 ArrayRef<unsigned> Indices,
1564 unsigned Factor) const override;
1565
1566 /// Lower interleaved store(s) into target specific
1567 /// instructions/intrinsics.
1569 unsigned Factor) const override;
1570
1572 int JTI, SelectionDAG &DAG) const override;
1573
1574 Align getPrefLoopAlignment(MachineLoop *ML) const override;
1575
1576 EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {
1577 if (VT == MVT::f80)
1578 return EVT::getIntegerVT(Context, 96);
1579 return TargetLoweringBase::getTypeToTransformTo(Context, VT);
1580 }
1581
1582 protected:
1583 std::pair<const TargetRegisterClass *, uint8_t>
1585 MVT VT) const override;
1586
1587 private:
1588 /// Keep a reference to the X86Subtarget around so that we can
1589 /// make the right decision when generating code for different targets.
1590 const X86Subtarget &Subtarget;
1591
1592 /// A list of legal FP immediates.
1593 std::vector<APFloat> LegalFPImmediates;
1594
1595 /// Indicate that this x86 target can instruction
1596 /// select the specified FP immediate natively.
1597 void addLegalFPImmediate(const APFloat& Imm) {
1598 LegalFPImmediates.push_back(Imm);
1599 }
1600
1601 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
1602 CallingConv::ID CallConv, bool isVarArg,
1603 const SmallVectorImpl<ISD::InputArg> &Ins,
1604 const SDLoc &dl, SelectionDAG &DAG,
1605 SmallVectorImpl<SDValue> &InVals,
1606 uint32_t *RegMask) const;
1607 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1608 const SmallVectorImpl<ISD::InputArg> &ArgInfo,
1609 const SDLoc &dl, SelectionDAG &DAG,
1610 const CCValAssign &VA, MachineFrameInfo &MFI,
1611 unsigned i) const;
1612 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1613 const SDLoc &dl, SelectionDAG &DAG,
1614 const CCValAssign &VA,
1615 ISD::ArgFlagsTy Flags, bool isByval) const;
1616
1617 // Call lowering helpers.
1618
1619 /// Check whether the call is eligible for tail call optimization. Targets
1620 /// that want to do tail call optimization should implement this function.
1621 bool IsEligibleForTailCallOptimization(
1622 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
1623 SmallVectorImpl<CCValAssign> &ArgLocs, bool IsCalleePopSRet) const;
1624 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
1625 SDValue Chain, bool IsTailCall,
1626 bool Is64Bit, int FPDiff,
1627 const SDLoc &dl) const;
1628
1629 unsigned GetAlignedArgumentStackSize(unsigned StackSize,
1630 SelectionDAG &DAG) const;
1631
1632 unsigned getAddressSpace() const;
1633
1634 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned,
1635 SDValue &Chain) const;
1636 SDValue LRINT_LLRINTHelper(SDNode *N, SelectionDAG &DAG) const;
1637
1638 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1639 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
1640 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1641 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1642
1643 unsigned getGlobalWrapperKind(const GlobalValue *GV,
1644 const unsigned char OpFlags) const;
1645 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1646 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1647 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1648 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1649 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
1650
1651 /// Creates target global address or external symbol nodes for calls or
1652 /// other uses.
1653 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1654 bool ForCall) const;
1655
1656 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1657 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1658 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1659 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1660 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
1661 SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;
1662 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1663 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1664 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1665 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
1666 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1667 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1668 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1669 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1670 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1671 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1672 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1673 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1674 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
1675 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1676 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1677 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
1678 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1679 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1680 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1681 SDValue LowerGET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
1682 SDValue LowerSET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
1683 SDValue LowerRESET_FPENV(SDValue Op, SelectionDAG &DAG) const;
1684 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
1685 SDValue LowerWin64_FP_TO_INT128(SDValue Op, SelectionDAG &DAG,
1686 SDValue &Chain) const;
1687 SDValue LowerWin64_INT128_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1688 SDValue LowerGC_TRANSITION(SDValue Op, SelectionDAG &DAG) const;
1689 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1690 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
1691 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1692 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
1693 SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
1694
1695 SDValue
1696 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1697 const SmallVectorImpl<ISD::InputArg> &Ins,
1698 const SDLoc &dl, SelectionDAG &DAG,
1699 SmallVectorImpl<SDValue> &InVals) const override;
1700 SDValue LowerCall(CallLoweringInfo &CLI,
1701 SmallVectorImpl<SDValue> &InVals) const override;
1702
1703 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1704 const SmallVectorImpl<ISD::OutputArg> &Outs,
1705 const SmallVectorImpl<SDValue> &OutVals,
1706 const SDLoc &dl, SelectionDAG &DAG) const override;
1707
1708 bool supportSplitCSR(MachineFunction *MF) const override {
1709 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
1710 MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
1711 }
1712 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
1713 void insertCopiesSplitCSR(
1714 MachineBasicBlock *Entry,
1715 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
1716
1717 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
1718
1719 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
1720
1721 EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
1722 ISD::NodeType ExtendKind) const override;
1723
1724 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
1725 bool isVarArg,
1726 const SmallVectorImpl<ISD::OutputArg> &Outs,
1727 LLVMContext &Context) const override;
1728
1729 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
1730 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;
1731
1733 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
1735 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
1737 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
1739 shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const;
1740 void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1741 void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1742
1743 LoadInst *
1744 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
1745
1746 bool needsCmpXchgNb(Type *MemType) const;
1747
1748 void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
1749 MachineBasicBlock *DispatchBB, int FI) const;
1750
1751 // Utility function to emit the low-level va_arg code for X86-64.
1752 MachineBasicBlock *
1753 EmitVAARGWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
1754
1755 /// Utility function to emit the xmm reg save portion of va_start.
1756 MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,
1757 MachineInstr &MI2,
1758 MachineBasicBlock *BB) const;
1759
1760 MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,
1761 MachineBasicBlock *BB) const;
1762
1763 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
1764 MachineBasicBlock *BB) const;
1765
1766 MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,
1767 MachineBasicBlock *BB) const;
1768
1769 MachineBasicBlock *EmitLoweredProbedAlloca(MachineInstr &MI,
1770 MachineBasicBlock *BB) const;
1771
1772 MachineBasicBlock *EmitLoweredTLSAddr(MachineInstr &MI,
1773 MachineBasicBlock *BB) const;
1774
1775 MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
1776 MachineBasicBlock *BB) const;
1777
1778 MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
1779 MachineBasicBlock *BB) const;
1780
1781 MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
1782 MachineBasicBlock *MBB) const;
1783
1784 void emitSetJmpShadowStackFix(MachineInstr &MI,
1785 MachineBasicBlock *MBB) const;
1786
1787 MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
1788 MachineBasicBlock *MBB) const;
1789
1790 MachineBasicBlock *emitLongJmpShadowStackFix(MachineInstr &MI,
1791 MachineBasicBlock *MBB) const;
1792
1793 MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,
1794 MachineBasicBlock *MBB) const;
1795
1796 MachineBasicBlock *emitPatchableEventCall(MachineInstr &MI,
1797 MachineBasicBlock *MBB) const;
1798
1799 /// Emit flags for the given setcc condition and operands. Also returns the
1800 /// corresponding X86 condition code constant in X86CC.
1801 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1, ISD::CondCode CC,
1802 const SDLoc &dl, SelectionDAG &DAG,
1803 SDValue &X86CC) const;
1804
1805 bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst,
1806 SDValue IntPow2) const override;
1807
1808 /// Check if replacement of SQRT with RSQRT should be disabled.
1809 bool isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const override;
1810
1811 /// Use rsqrt* to speed up sqrt calculations.
1812 SDValue getSqrtEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
1813 int &RefinementSteps, bool &UseOneConstNR,
1814 bool Reciprocal) const override;
1815
1816 /// Use rcp* to speed up fdiv calculations.
1817 SDValue getRecipEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
1818 int &RefinementSteps) const override;
1819
1820 /// Reassociate floating point divisions into multiply by reciprocal.
1821 unsigned combineRepeatedFPDivisors() const override;
1822
1823 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1824 SmallVectorImpl<SDNode *> &Created) const override;
1825
1826 SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
1827 SDValue V2) const;
1828 };
1829
1830 namespace X86 {
1831 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
1832 const TargetLibraryInfo *libInfo);
1833 } // end namespace X86
1834
1835 // X86 specific Gather/Scatter nodes.
1836 // The class has the same order of operands as MaskedGatherScatterSDNode for
1837 // convenience.
1839 public:
1840 // This is a intended as a utility and should never be directly created.
1843
1844 const SDValue &getBasePtr() const { return getOperand(3); }
1845 const SDValue &getIndex() const { return getOperand(4); }
1846 const SDValue &getMask() const { return getOperand(2); }
1847 const SDValue &getScale() const { return getOperand(5); }
1848
1849 static bool classof(const SDNode *N) {
1850 return N->getOpcode() == X86ISD::MGATHER ||
1851 N->getOpcode() == X86ISD::MSCATTER;
1852 }
1853 };
1854
1856 public:
1857 const SDValue &getPassThru() const { return getOperand(1); }
1858
1859 static bool classof(const SDNode *N) {
1860 return N->getOpcode() == X86ISD::MGATHER;
1861 }
1862 };
1863
1865 public:
1866 const SDValue &getValue() const { return getOperand(1); }
1867
1868 static bool classof(const SDNode *N) {
1869 return N->getOpcode() == X86ISD::MSCATTER;
1870 }
1871 };
1872
1873 /// Generate unpacklo/unpackhi shuffle mask.
1874 void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
1875 bool Unary);
1876
1877 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
1878 /// imposed by AVX and specific to the unary pattern. Example:
1879 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
1880 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
1881 void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);
1882
1883} // end namespace llvm
1884
1885#endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
BlockVerifier::State From
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
uint64_t Addr
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define RegName(no)
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
This file describes how to lower LLVM code to machine code.
static bool is64Bit(const char *name)
Class for arbitrary precision integers.
Definition: APInt.h:77
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:185
This class is used to represent ISD::LOAD nodes.
Context object for machine code objects.
Definition: MCContext.h:81
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
Machine Value Type.
static MVT getIntegerVT(unsigned BitWidth)
Instructions::iterator instr_iterator
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const SDValue & getOperand(unsigned Num) const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:227
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
An instruction for storing to memory.
Definition: Instructions.h:318
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
NegatibleCost
Enum that specifies when a float negation is beneficial.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
const SDValue & getPassThru() const
static bool classof(const SDNode *N)
const SDValue & getBasePtr() const
const SDValue & getScale() const
static bool classof(const SDNode *N)
const SDValue & getIndex() const
const SDValue & getValue() const
static bool classof(const SDNode *N)
bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override
Overflow nodes should get combined/lowered to optimal instructions (they should allow eliminating exp...
Align getPrefLoopAlignment(MachineLoop *ML) const override
Return the preferred loop alignment.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const override
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const override
Returns relocation base for the given PIC jumptable.
bool preferABDSToABSWithNSW(EVT VT) const override
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
std::pair< SDValue, SDValue > BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer, MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool isMemoryAccessFast(EVT VT, Align Alignment) const
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, const SDLoc &DL, const AsmOperandInfo &Constraint, SelectionDAG &DAG) const override
Handle Lowering flag assembly outputs.
bool supportKCFIBundles() const override
Return true if the target supports kcfi operand bundles.
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const override
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth) const override
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
bool convertSelectOfConstantsToMath(EVT VT) const override
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint letter, return the type of constraint for this target.
bool hasVectorBlend() const override
Return true if the target has a vector blend instruction.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
bool useSoftFloat() const override
bool isVectorShiftByScalarCheap(Type *Ty) const override
This is used to enable splatted operand transforms for vector shifts and vector funnel shifts.
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool isLegalStoreImmediate(int64_t Imm) const override
Return true if the specified immediate is legal for the value input of a store instruction.
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const override
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
bool isCtlzFast() const override
Return true if ctlz instruction is fast.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool isSafeMemOpType(MVT VT) const override
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const override
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool shouldSplatInsEltVarIndex(EVT VT) const override
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Return true if sinking I's operands to the same basic block as I is profitable, e....
bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const override
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
MVT hasFastEqualityCompare(unsigned NumBits) const override
Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
It returns EVT::Other if the type should be determined using generic target-independent logic.
bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op, const APInt &DemandedElts, unsigned MaskIndex, TargetLoweringOpt &TLO, unsigned Depth) const
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const override
bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond, EVT VT) const override
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if we believe it is correct and profitable to reduce the load node to a smaller type.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, const MachineMemOperand &MMO, unsigned *Fast) const
bool preferScalarizeSplat(SDNode *N) const override
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
Returns true if the target allows unaligned memory accesses of the specified type.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const override
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower interleaved load(s) into target specific instructions/intrinsics.
StringRef getStackProbeSymbolName(const MachineFunction &MF) const override
Returns the name of the symbol used to emit stack probes or the empty string if not applicable.
uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override
Return the desired alignment for ByVal aggregate function arguments in the caller parameter area.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isShuffleMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
bool useStackGuardXorFP() const override
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
bool shouldScalarizeBinop(SDValue) const override
Scalar ops always have equal or better analysis/performance/power than the vector equivalent,...
void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const override
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type Ty1 to type Ty2.
Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const override
Return true if the target stores SafeStack pointer at a fixed offset in some non-standard address spa...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool areJTsAllowed(const Function *Fn) const override
Returns true if lowering to a jump table is allowed.
bool isCommutativeBinOp(unsigned Opcode) const override
Returns true if the opcode is a commutative binary operation.
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const override
Returns preferred type for switch condition.
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower interleaved store(s) into target specific instructions/intrinsics.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool convertSetCCLogicToBitwiseLogic(EVT VT) const override
Use bitwise logic to make pairs of compares more efficient.
bool isVectorClearMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Similar to isShuffleMaskLegal.
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &Info, const char *Constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Customize the preferred legalization strategy for certain types.
bool shouldConvertPhiType(Type *From, Type *To) const override
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
bool hasStackProbeSymbol(const MachineFunction &MF) const override
Returns true if stack probing through a function call is requested.
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type Ty1 implicit zero-extends the valu...
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isTargetCanonicalConstantNode(SDValue Op) const override
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool softPromoteHalfType() const override
SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const override
bool mergeStoresAfterLegalization(EVT MemVT) const override
Do not merge vector stores after legalization because that may conflict with x86-specific store split...
TargetLowering::AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const override
Return prefered fold type, Abs if this is a vector, AddAnd if its an integer, None otherwise.
bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool addressingModeSupportsTLS(const GlobalValue &GV) const override
Returns true if the targets addressing mode can target thread local storage (TLS).
SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isBinOp(unsigned Opcode) const override
Add x86-specific opcodes to the default list.
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDValue unwrapAddress(SDValue N) const override
CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the value type to use for ISD::SETCC.
bool isVectorLoadExtDesirable(SDValue) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override
For types supported by the target, this is an identity function.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
unsigned getStackProbeSize(const MachineFunction &MF) const
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
Replace the results of node with an illegal result type with new values built out of custom code.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool needsFixedCatchObjects() const override
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Extract of a scalar FP value from index 0 of a vector is free.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ CXX_FAST_TLS
Used for access functions.
Definition: CallingConv.h:72
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:561
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:905
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1419
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:575
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:858
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1431
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
Definition: ISDOpcodes.h:1425
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1542
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1522
@ X86
Windows x64, Windows Itanium (IA-64)
@ FST
This instruction implements a truncating store from FP stack slots.
@ REP_MOVS
Repeat move, corresponds to X86::REP_MOVSx.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ FMAX
Floating point max and min.
@ BT
X86 bit-test instructions.
@ HADD
Integer horizontal add/sub.
@ MOVQ2DQ
Copies a 64-bit value from an MMX vector to the low word of an XMM vector, with the high word zero fi...
@ BLENDI
Blend where the selector is an immediate.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ ADDSUB
Combined add and sub on an FP vector.
@ RET_GLUE
Return with a glue operand.
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ FHADD
Floating point horizontal add/sub.
@ FMAXS
Scalar intrinsic floating point max and min.
@ BSR
Bit scan reverse.
@ IRET
Return from interrupt. Operand 0 is the number of bytes to pop.
@ SETCC
X86 SetCC.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ SELECTS
X86 Select.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ PEXTRB
Extract an 8-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRB.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ FSETCC
X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
@ PINSRB
Insert the lower 8-bits of a 32-bit value to a vector, corresponds to X86::PINSRB.
@ REP_STOS
Repeat fill, corresponds to X86::REP_STOSx.
@ INSERTPS
Insert any element of a 4 x float vector into any element of a destination 4 x floatvector.
@ PSHUFB
Shuffle 16 8-bit values within a vector.
@ PEXTRW
Extract a 16-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRW.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ AADD
RAO arithmetic instructions.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FMAXC
Commutative FMIN and FMAX.
@ EXTRQI
SSE4A Extraction and Insertion.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ PSADBW
Compute Sum of Absolute Differences.
@ FOR
Bitwise logical OR of floating point values.
@ FIST
This instruction implements a fp->int store from FP stack slots.
@ FP_TO_INT_IN_MEM
This instruction implements FP_TO_SINT with the integer destination in memory and a FP reg source.
@ LADD
LOCK-prefixed arithmetic read-modify-write instructions.
@ DBPSADBW
Compute Double Block Packed Sum-Absolute-Differences.
@ MMX_MOVW2D
Copies a GPR into the low 32-bit word of a MMX vector and zero out the high word.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ PINSRW
Insert the lower 16-bits of a 32-bit value to a vector, corresponds to X86::PINSRW.
@ CMPCCXADD
Compare and Add if Condition is Met.
@ NT_CALL
Same as call except it adds the NoTrack prefix.
@ MMX_MOVD2W
Copies a 32-bit value from the low word of a MMX vector to a GPR.
@ FILD
This instruction implements SINT_TO_FP with the integer source in memory and FP reg result.
@ MOVDQ2Q
Copies a 64-bit value from the low word of an XMM vector to an MMX vector.
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ BSF
Bit scan forward.
@ VAARG_64
These instructions grab the address of the next argument from a va_list.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
@ FSHL
X86 funnel/double shift i16 instructions.
@ FRSQRT
Floating point reciprocal-sqrt and reciprocal approximation.
RoundingMode
Current rounding mode is represented in bits 11:10 of FPSR.
bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into a vector splat instruction as a memory oper...
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
bool mayFoldIntoZeroExtend(SDValue Op)
Check if Op is an operation that could be folded into a zero extend x86 instruction.
bool mayFoldIntoStore(SDValue Op)
Check if Op is a value that could be used to fold a store into some other x86 instruction as a memory...
bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget, const MachineFunction &MF)
True if the target supports the extended frame for async Swift functions.
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool GuaranteeTCO)
Determines whether the callee is required to pop its own arguments.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
AddressSpace
Definition: NVPTXBaseInfo.h:21
void createUnpackShuffleMask(EVT VT, SmallVectorImpl< int > &Mask, bool Lo, bool Unary)
Generate unpacklo/unpackhi shuffle mask.
void createSplat2ShuffleMask(MVT VT, SmallVectorImpl< int > &Mask, bool Lo)
Similar to unpacklo/unpackhi, but without the 128-bit lane limitation imposed by AVX and specific to ...
CombineLevel
Definition: DAGCombine.h:15
DWARFExpression::Operation Op
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
This class contains a discriminated union of information about pointers in memory operands,...