LLVM 19.0.0git
AArch64SelectionDAGInfo.cpp
Go to the documentation of this file.
1//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64SelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
14using namespace llvm;
15
16#define DEBUG_TYPE "aarch64-selectiondag-info"
17
19 SelectionDAG &DAG, const SDLoc &DL,
20 SDValue Chain, SDValue Dst,
21 SDValue SrcOrValue, SDValue Size,
22 Align Alignment, bool isVolatile,
23 MachinePointerInfo DstPtrInfo,
24 MachinePointerInfo SrcPtrInfo) const {
25
26 // Get the constant size of the copy/set.
27 uint64_t ConstSize = 0;
28 if (auto *C = dyn_cast<ConstantSDNode>(Size))
29 ConstSize = C->getZExtValue();
30
31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
33
34 const auto MachineOpcode = [&]() {
35 switch (SDOpcode) {
37 return AArch64::MOPSMemorySetPseudo;
39 return AArch64::MOPSMemorySetTaggingPseudo;
41 return AArch64::MOPSMemoryCopyPseudo;
43 return AArch64::MOPSMemoryMovePseudo;
44 default:
45 llvm_unreachable("Unhandled MOPS ISD Opcode");
46 }
47 }();
48
50
51 auto Vol =
53 auto DstFlags = MachineMemOperand::MOStore | Vol;
54 auto *DstOp =
55 MF.getMachineMemOperand(DstPtrInfo, DstFlags, ConstSize, Alignment);
56
57 if (IsSet) {
58 // Extend value to i64, if required.
59 if (SrcOrValue.getValueType() != MVT::i64)
60 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
61 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
62 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
63 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
64 DAG.setNodeMemRefs(Node, {DstOp});
65 return SDValue(Node, 2);
66 } else {
67 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
68 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
69 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
70
71 auto SrcFlags = MachineMemOperand::MOLoad | Vol;
72 auto *SrcOp =
73 MF.getMachineMemOperand(SrcPtrInfo, SrcFlags, ConstSize, Alignment);
74 DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
75 return SDValue(Node, 3);
76 }
77}
78
80 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
81 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
82 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
83 const AArch64Subtarget &STI =
85 if (STI.hasMOPS())
86 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
87 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
88 return SDValue();
89}
90
92 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
93 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
94 MachinePointerInfo DstPtrInfo) const {
95 const AArch64Subtarget &STI =
97
98 if (STI.hasMOPS()) {
99 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
100 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
101 }
102 return SDValue();
103}
104
106 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
107 SDValue Size, Align Alignment, bool isVolatile,
108 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
109 const AArch64Subtarget &STI =
111 if (STI.hasMOPS()) {
112 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
113 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
114 }
115 return SDValue();
116}
117
118static const int kSetTagLoopThreshold = 176;
119
121 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
122 const MachineMemOperand *BaseMemOperand,
123 bool ZeroData) {
125 unsigned ObjSizeScaled = ObjSize / 16;
126
127 SDValue TagSrc = Ptr;
128 if (Ptr.getOpcode() == ISD::FrameIndex) {
129 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
130 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
131 // A frame index operand may end up as [SP + offset] => it is fine to use SP
132 // register as the tag source.
133 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
134 }
135
136 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
137 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
138
139 SmallVector<SDValue, 8> OutChains;
140 unsigned OffsetScaled = 0;
141 while (OffsetScaled < ObjSizeScaled) {
142 if (ObjSizeScaled - OffsetScaled >= 2) {
143 SDValue AddrNode = DAG.getMemBasePlusOffset(
144 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);
146 OpCode2, dl, DAG.getVTList(MVT::Other),
147 {Chain, TagSrc, AddrNode},
148 MVT::v4i64,
149 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
150 OffsetScaled += 2;
151 OutChains.push_back(St);
152 continue;
153 }
154
155 if (ObjSizeScaled - OffsetScaled > 0) {
156 SDValue AddrNode = DAG.getMemBasePlusOffset(
157 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);
159 OpCode1, dl, DAG.getVTList(MVT::Other),
160 {Chain, TagSrc, AddrNode},
161 MVT::v2i64,
162 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
163 OffsetScaled += 1;
164 OutChains.push_back(St);
165 }
166 }
167
168 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
169 return Res;
170}
171
173 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
174 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
175 uint64_t ObjSize = Size->getAsZExtVal();
176 assert(ObjSize % 16 == 0);
177
179 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
180 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
181
182 bool UseSetTagRangeLoop =
183 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
184 if (!UseSetTagRangeLoop)
185 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
186 ZeroData);
187
188 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
189
190 unsigned Opcode;
191 if (Addr.getOpcode() == ISD::FrameIndex) {
192 int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
193 Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
194 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
195 } else {
196 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
197 }
198 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
199 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
200
201 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
202 return SDValue(St, 2);
203}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Ptr, uint64_t ObjSize, const MachineMemOperand *BaseMemOperand, bool ZeroData)
static const int kSetTagLoopThreshold
uint64_t Addr
uint64_t Size
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memmove.
SDValue EmitMOPS(AArch64ISD::NodeType SDOpcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo DstPtrInfo, bool ZeroData) const override
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
EVT getValueType() const
Return the ValueType of the referenced return value.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:332
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FrameIndex
Definition: ISDOpcodes.h:80
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
This class contains a discriminated union of information about pointers in memory operands,...