Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name PPCISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/PowerPC -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/PowerPC -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/PowerPC -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-08-28-193554-24367-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp

1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the PPCISelLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCISelLowering.h"
14#include "MCTargetDesc/PPCPredicates.h"
15#include "PPC.h"
16#include "PPCCCState.h"
17#include "PPCCallingConv.h"
18#include "PPCFrameLowering.h"
19#include "PPCInstrInfo.h"
20#include "PPCMachineFunctionInfo.h"
21#include "PPCPerfectShuffle.h"
22#include "PPCRegisterInfo.h"
23#include "PPCSubtarget.h"
24#include "PPCTargetMachine.h"
25#include "llvm/ADT/APFloat.h"
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/None.h"
30#include "llvm/ADT/STLExtras.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/ADT/StringRef.h"
36#include "llvm/ADT/StringSwitch.h"
37#include "llvm/CodeGen/CallingConvLower.h"
38#include "llvm/CodeGen/ISDOpcodes.h"
39#include "llvm/CodeGen/MachineBasicBlock.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachineJumpTableInfo.h"
45#include "llvm/CodeGen/MachineLoopInfo.h"
46#include "llvm/CodeGen/MachineMemOperand.h"
47#include "llvm/CodeGen/MachineModuleInfo.h"
48#include "llvm/CodeGen/MachineOperand.h"
49#include "llvm/CodeGen/MachineRegisterInfo.h"
50#include "llvm/CodeGen/RuntimeLibcalls.h"
51#include "llvm/CodeGen/SelectionDAG.h"
52#include "llvm/CodeGen/SelectionDAGNodes.h"
53#include "llvm/CodeGen/TargetInstrInfo.h"
54#include "llvm/CodeGen/TargetLowering.h"
55#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
56#include "llvm/CodeGen/TargetRegisterInfo.h"
57#include "llvm/CodeGen/ValueTypes.h"
58#include "llvm/IR/CallingConv.h"
59#include "llvm/IR/Constant.h"
60#include "llvm/IR/Constants.h"
61#include "llvm/IR/DataLayout.h"
62#include "llvm/IR/DebugLoc.h"
63#include "llvm/IR/DerivedTypes.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/GlobalValue.h"
66#include "llvm/IR/IRBuilder.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/Intrinsics.h"
69#include "llvm/IR/IntrinsicsPowerPC.h"
70#include "llvm/IR/Module.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/MC/MCExpr.h"
76#include "llvm/MC/MCRegisterInfo.h"
77#include "llvm/MC/MCSectionXCOFF.h"
78#include "llvm/MC/MCSymbolXCOFF.h"
79#include "llvm/Support/AtomicOrdering.h"
80#include "llvm/Support/BranchProbability.h"
81#include "llvm/Support/Casting.h"
82#include "llvm/Support/CodeGen.h"
83#include "llvm/Support/CommandLine.h"
84#include "llvm/Support/Compiler.h"
85#include "llvm/Support/Debug.h"
86#include "llvm/Support/ErrorHandling.h"
87#include "llvm/Support/Format.h"
88#include "llvm/Support/KnownBits.h"
89#include "llvm/Support/MachineValueType.h"
90#include "llvm/Support/MathExtras.h"
91#include "llvm/Support/raw_ostream.h"
92#include "llvm/Target/TargetMachine.h"
93#include "llvm/Target/TargetOptions.h"
94#include <algorithm>
95#include <cassert>
96#include <cstdint>
97#include <iterator>
98#include <list>
99#include <utility>
100#include <vector>
101
102using namespace llvm;
103
104#define DEBUG_TYPE"ppc-lowering" "ppc-lowering"
105
106static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
107cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
108
109static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref",
110cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden);
111
112static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned",
113cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden);
114
115static cl::opt<bool> DisableSCO("disable-ppc-sco",
116cl::desc("disable sibling call optimization on ppc"), cl::Hidden);
117
118static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32",
119cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden);
120
121static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
122cl::desc("use absolute jump tables on ppc"), cl::Hidden);
123
124static cl::opt<bool> EnableQuadwordAtomics(
125 "ppc-quadword-atomics",
126 cl::desc("enable quadword lock-free atomic operations"), cl::init(false),
127 cl::Hidden);
128
129STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"ppc-lowering", "NumTailCalls"
, "Number of tail calls"}
;
130STATISTIC(NumSiblingCalls, "Number of sibling calls")static llvm::Statistic NumSiblingCalls = {"ppc-lowering", "NumSiblingCalls"
, "Number of sibling calls"}
;
131STATISTIC(ShufflesHandledWithVPERM, "Number of shuffles lowered to a VPERM")static llvm::Statistic ShufflesHandledWithVPERM = {"ppc-lowering"
, "ShufflesHandledWithVPERM", "Number of shuffles lowered to a VPERM"
}
;
132STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed")static llvm::Statistic NumDynamicAllocaProbed = {"ppc-lowering"
, "NumDynamicAllocaProbed", "Number of dynamic stack allocation probed"
}
;
133
134static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
135
136static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
137
138static const char AIXSSPCanaryWordName[] = "__ssp_canary_word";
139
140// FIXME: Remove this once the bug has been fixed!
141extern cl::opt<bool> ANDIGlueBug;
142
143PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
144 const PPCSubtarget &STI)
145 : TargetLowering(TM), Subtarget(STI) {
146 // Initialize map that relates the PPC addressing modes to the computed flags
147 // of a load/store instruction. The map is used to determine the optimal
148 // addressing mode when selecting load and stores.
149 initializeAddrModeMap();
150 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
151 // arguments are at least 4/8 bytes aligned.
152 bool isPPC64 = Subtarget.isPPC64();
153 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
154
155 // Set up the register classes.
156 addRegisterClass(MVT::i32, &PPC::GPRCRegClass);
157 if (!useSoftFloat()) {
158 if (hasSPE()) {
159 addRegisterClass(MVT::f32, &PPC::GPRCRegClass);
160 // EFPU2 APU only supports f32
161 if (!Subtarget.hasEFPU2())
162 addRegisterClass(MVT::f64, &PPC::SPERCRegClass);
163 } else {
164 addRegisterClass(MVT::f32, &PPC::F4RCRegClass);
165 addRegisterClass(MVT::f64, &PPC::F8RCRegClass);
166 }
167 }
168
169 // Match BITREVERSE to customized fast code sequence in the td file.
170 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
171 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
172
173 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended.
174 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
175
176 // Custom lower inline assembly to check for special registers.
177 setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
178 setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom);
179
180 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD.
181 for (MVT VT : MVT::integer_valuetypes()) {
182 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
183 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand);
184 }
185
186 if (Subtarget.isISA3_0()) {
187 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Legal);
188 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Legal);
189 setTruncStoreAction(MVT::f64, MVT::f16, Legal);
190 setTruncStoreAction(MVT::f32, MVT::f16, Legal);
191 } else {
192 // No extending loads from f16 or HW conversions back and forth.
193 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
194 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
195 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
196 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
197 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
198 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
199 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
200 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
201 }
202
203 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
204
205 // PowerPC has pre-inc load and store's.
206 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal);
207 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal);
208 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal);
209 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal);
210 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal);
211 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal);
212 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal);
213 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal);
214 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal);
215 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal);
216 if (!Subtarget.hasSPE()) {
217 setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal);
218 setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal);
219 setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal);
220 setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal);
221 }
222
223 // PowerPC uses ADDC/ADDE/SUBC/SUBE to propagate carry.
224 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
225 for (MVT VT : ScalarIntVTs) {
226 setOperationAction(ISD::ADDC, VT, Legal);
227 setOperationAction(ISD::ADDE, VT, Legal);
228 setOperationAction(ISD::SUBC, VT, Legal);
229 setOperationAction(ISD::SUBE, VT, Legal);
230 }
231
232 if (Subtarget.useCRBits()) {
233 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
234
235 if (isPPC64 || Subtarget.hasFPCVT()) {
236 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Promote);
237 AddPromotedToType(ISD::STRICT_SINT_TO_FP, MVT::i1,
238 isPPC64 ? MVT::i64 : MVT::i32);
239 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Promote);
240 AddPromotedToType(ISD::STRICT_UINT_TO_FP, MVT::i1,
241 isPPC64 ? MVT::i64 : MVT::i32);
242
243 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);
244 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1,
245 isPPC64 ? MVT::i64 : MVT::i32);
246 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
247 AddPromotedToType(ISD::UINT_TO_FP, MVT::i1,
248 isPPC64 ? MVT::i64 : MVT::i32);
249
250 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i1, Promote);
251 AddPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::i1,
252 isPPC64 ? MVT::i64 : MVT::i32);
253 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i1, Promote);
254 AddPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::i1,
255 isPPC64 ? MVT::i64 : MVT::i32);
256
257 setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);
258 AddPromotedToType(ISD::FP_TO_SINT, MVT::i1,
259 isPPC64 ? MVT::i64 : MVT::i32);
260 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
261 AddPromotedToType(ISD::FP_TO_UINT, MVT::i1,
262 isPPC64 ? MVT::i64 : MVT::i32);
263 } else {
264 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Custom);
265 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Custom);
266 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom);
267 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom);
268 }
269
270 // PowerPC does not support direct load/store of condition registers.
271 setOperationAction(ISD::LOAD, MVT::i1, Custom);
272 setOperationAction(ISD::STORE, MVT::i1, Custom);
273
274 // FIXME: Remove this once the ANDI glue bug is fixed:
275 if (ANDIGlueBug)
276 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom);
277
278 for (MVT VT : MVT::integer_valuetypes()) {
279 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
280 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
281 setTruncStoreAction(VT, MVT::i1, Expand);
282 }
283
284 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass);
285 }
286
287 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
288 // PPC (the libcall is not available).
289 setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom);
290 setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom);
291 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::ppcf128, Custom);
292 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::ppcf128, Custom);
293
294 // We do not currently implement these libm ops for PowerPC.
295 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand);
296 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand);
297 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand);
298 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand);
299 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand);
300 setOperationAction(ISD::FREM, MVT::ppcf128, Expand);
301
302 // PowerPC has no SREM/UREM instructions unless we are on P9
303 // On P9 we may use a hardware instruction to compute the remainder.
304 // When the result of both the remainder and the division is required it is
305 // more efficient to compute the remainder from the result of the division
306 // rather than use the remainder instruction. The instructions are legalized
307 // directly because the DivRemPairsPass performs the transformation at the IR
308 // level.
309 if (Subtarget.isISA3_0()) {
310 setOperationAction(ISD::SREM, MVT::i32, Legal);
311 setOperationAction(ISD::UREM, MVT::i32, Legal);
312 setOperationAction(ISD::SREM, MVT::i64, Legal);
313 setOperationAction(ISD::UREM, MVT::i64, Legal);
314 } else {
315 setOperationAction(ISD::SREM, MVT::i32, Expand);
316 setOperationAction(ISD::UREM, MVT::i32, Expand);
317 setOperationAction(ISD::SREM, MVT::i64, Expand);
318 setOperationAction(ISD::UREM, MVT::i64, Expand);
319 }
320
321 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
322 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
323 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
324 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
325 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
326 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
327 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
328 setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
329 setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
330
331 // Handle constrained floating-point operations of scalar.
332 // TODO: Handle SPE specific operation.
333 setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
334 setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
335 setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
336 setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
337 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
338
339 setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
340 setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
341 setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
342 setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
343
344 if (!Subtarget.hasSPE()) {
345 setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
346 setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
347 }
348
349 if (Subtarget.hasVSX()) {
350 setOperationAction(ISD::STRICT_FRINT, MVT::f32, Legal);
351 setOperationAction(ISD::STRICT_FRINT, MVT::f64, Legal);
352 }
353
354 if (Subtarget.hasFSQRT()) {
355 setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
356 setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
357 }
358
359 if (Subtarget.hasFPRND()) {
360 setOperationAction(ISD::STRICT_FFLOOR, MVT::f32, Legal);
361 setOperationAction(ISD::STRICT_FCEIL, MVT::f32, Legal);
362 setOperationAction(ISD::STRICT_FTRUNC, MVT::f32, Legal);
363 setOperationAction(ISD::STRICT_FROUND, MVT::f32, Legal);
364
365 setOperationAction(ISD::STRICT_FFLOOR, MVT::f64, Legal);
366 setOperationAction(ISD::STRICT_FCEIL, MVT::f64, Legal);
367 setOperationAction(ISD::STRICT_FTRUNC, MVT::f64, Legal);
368 setOperationAction(ISD::STRICT_FROUND, MVT::f64, Legal);
369 }
370
371 // We don't support sin/cos/sqrt/fmod/pow
372 setOperationAction(ISD::FSIN , MVT::f64, Expand);
373 setOperationAction(ISD::FCOS , MVT::f64, Expand);
374 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
375 setOperationAction(ISD::FREM , MVT::f64, Expand);
376 setOperationAction(ISD::FPOW , MVT::f64, Expand);
377 setOperationAction(ISD::FSIN , MVT::f32, Expand);
378 setOperationAction(ISD::FCOS , MVT::f32, Expand);
379 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
380 setOperationAction(ISD::FREM , MVT::f32, Expand);
381 setOperationAction(ISD::FPOW , MVT::f32, Expand);
382 if (Subtarget.hasSPE()) {
383 setOperationAction(ISD::FMA , MVT::f64, Expand);
384 setOperationAction(ISD::FMA , MVT::f32, Expand);
385 } else {
386 setOperationAction(ISD::FMA , MVT::f64, Legal);
387 setOperationAction(ISD::FMA , MVT::f32, Legal);
388 }
389
390 if (Subtarget.hasSPE())
391 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
392
393 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
394
395 // If we're enabling GP optimizations, use hardware square root
396 if (!Subtarget.hasFSQRT() &&
397 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() &&
398 Subtarget.hasFRE()))
399 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
400
401 if (!Subtarget.hasFSQRT() &&
402 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() &&
403 Subtarget.hasFRES()))
404 setOperationAction(ISD::FSQRT, MVT::f32, Expand);
405
406 if (Subtarget.hasFCPSGN()) {
407 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal);
408 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal);
409 } else {
410 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
411 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
412 }
413
414 if (Subtarget.hasFPRND()) {
415 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
416 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
417 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
418 setOperationAction(ISD::FROUND, MVT::f64, Legal);
419
420 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
421 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
422 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
423 setOperationAction(ISD::FROUND, MVT::f32, Legal);
424 }
425
426 // PowerPC does not have BSWAP, but we can use vector BSWAP instruction xxbrd
427 // to speed up scalar BSWAP64.
428 // CTPOP or CTTZ were introduced in P8/P9 respectively
429 setOperationAction(ISD::BSWAP, MVT::i32 , Expand);
430 if (Subtarget.hasP9Vector() && Subtarget.isPPC64())
431 setOperationAction(ISD::BSWAP, MVT::i64 , Custom);
432 else
433 setOperationAction(ISD::BSWAP, MVT::i64 , Expand);
434 if (Subtarget.isISA3_0()) {
435 setOperationAction(ISD::CTTZ , MVT::i32 , Legal);
436 setOperationAction(ISD::CTTZ , MVT::i64 , Legal);
437 } else {
438 setOperationAction(ISD::CTTZ , MVT::i32 , Expand);
439 setOperationAction(ISD::CTTZ , MVT::i64 , Expand);
440 }
441
442 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
443 setOperationAction(ISD::CTPOP, MVT::i32 , Legal);
444 setOperationAction(ISD::CTPOP, MVT::i64 , Legal);
445 } else {
446 setOperationAction(ISD::CTPOP, MVT::i32 , Expand);
447 setOperationAction(ISD::CTPOP, MVT::i64 , Expand);
448 }
449
450 // PowerPC does not have ROTR
451 setOperationAction(ISD::ROTR, MVT::i32 , Expand);
452 setOperationAction(ISD::ROTR, MVT::i64 , Expand);
453
454 if (!Subtarget.useCRBits()) {
455 // PowerPC does not have Select
456 setOperationAction(ISD::SELECT, MVT::i32, Expand);
457 setOperationAction(ISD::SELECT, MVT::i64, Expand);
458 setOperationAction(ISD::SELECT, MVT::f32, Expand);
459 setOperationAction(ISD::SELECT, MVT::f64, Expand);
460 }
461
462 // PowerPC wants to turn select_cc of FP into fsel when possible.
463 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
464 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
465
466 // PowerPC wants to optimize integer setcc a bit
467 if (!Subtarget.useCRBits())
468 setOperationAction(ISD::SETCC, MVT::i32, Custom);
469
470 if (Subtarget.hasFPU()) {
471 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
472 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
473 setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Legal);
474
475 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
476 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
477 setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Legal);
478 }
479
480 // PowerPC does not have BRCOND which requires SetCC
481 if (!Subtarget.useCRBits())
482 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
483
484 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
485
486 if (Subtarget.hasSPE()) {
487 // SPE has built-in conversions
488 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Legal);
489 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Legal);
490 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Legal);
491 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
492 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
493 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
494
495 // SPE supports signaling compare of f32/f64.
496 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
497 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
498 } else {
499 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
500 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
501 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
502
503 // PowerPC does not have [U|S]INT_TO_FP
504 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Expand);
505 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Expand);
506 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand);
507 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand);
508 }
509
510 if (Subtarget.hasDirectMove() && isPPC64) {
511 setOperationAction(ISD::BITCAST, MVT::f32, Legal);
512 setOperationAction(ISD::BITCAST, MVT::i32, Legal);
513 setOperationAction(ISD::BITCAST, MVT::i64, Legal);
514 setOperationAction(ISD::BITCAST, MVT::f64, Legal);
515 if (TM.Options.UnsafeFPMath) {
516 setOperationAction(ISD::LRINT, MVT::f64, Legal);
517 setOperationAction(ISD::LRINT, MVT::f32, Legal);
518 setOperationAction(ISD::LLRINT, MVT::f64, Legal);
519 setOperationAction(ISD::LLRINT, MVT::f32, Legal);
520 setOperationAction(ISD::LROUND, MVT::f64, Legal);
521 setOperationAction(ISD::LROUND, MVT::f32, Legal);
522 setOperationAction(ISD::LLROUND, MVT::f64, Legal);
523 setOperationAction(ISD::LLROUND, MVT::f32, Legal);
524 }
525 } else {
526 setOperationAction(ISD::BITCAST, MVT::f32, Expand);
527 setOperationAction(ISD::BITCAST, MVT::i32, Expand);
528 setOperationAction(ISD::BITCAST, MVT::i64, Expand);
529 setOperationAction(ISD::BITCAST, MVT::f64, Expand);
530 }
531
532 // We cannot sextinreg(i1). Expand to shifts.
533 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
534
535 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
536 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
537 // support continuation, user-level threading, and etc.. As a result, no
538 // other SjLj exception interfaces are implemented and please don't build
539 // your own exception handling based on them.
540 // LLVM/Clang supports zero-cost DWARF exception handling.
541 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
542 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
543
544 // We want to legalize GlobalAddress and ConstantPool nodes into the
545 // appropriate instructions to materialize the address.
546 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
547 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
548 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
549 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
550 setOperationAction(ISD::JumpTable, MVT::i32, Custom);
551 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
552 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
553 setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
554 setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
555 setOperationAction(ISD::JumpTable, MVT::i64, Custom);
556
557 // TRAP is legal.
558 setOperationAction(ISD::TRAP, MVT::Other, Legal);
559
560 // TRAMPOLINE is custom lowered.
561 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
562 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
563
564 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
565 setOperationAction(ISD::VASTART , MVT::Other, Custom);
566
567 if (Subtarget.is64BitELFABI()) {
568 // VAARG always uses double-word chunks, so promote anything smaller.
569 setOperationAction(ISD::VAARG, MVT::i1, Promote);
570 AddPromotedToType(ISD::VAARG, MVT::i1, MVT::i64);
571 setOperationAction(ISD::VAARG, MVT::i8, Promote);
572 AddPromotedToType(ISD::VAARG, MVT::i8, MVT::i64);
573 setOperationAction(ISD::VAARG, MVT::i16, Promote);
574 AddPromotedToType(ISD::VAARG, MVT::i16, MVT::i64);
575 setOperationAction(ISD::VAARG, MVT::i32, Promote);
576 AddPromotedToType(ISD::VAARG, MVT::i32, MVT::i64);
577 setOperationAction(ISD::VAARG, MVT::Other, Expand);
578 } else if (Subtarget.is32BitELFABI()) {
579 // VAARG is custom lowered with the 32-bit SVR4 ABI.
580 setOperationAction(ISD::VAARG, MVT::Other, Custom);
581 setOperationAction(ISD::VAARG, MVT::i64, Custom);
582 } else
583 setOperationAction(ISD::VAARG, MVT::Other, Expand);
584
585 // VACOPY is custom lowered with the 32-bit SVR4 ABI.
586 if (Subtarget.is32BitELFABI())
587 setOperationAction(ISD::VACOPY , MVT::Other, Custom);
588 else
589 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
590
591 // Use the default implementation.
592 setOperationAction(ISD::VAEND , MVT::Other, Expand);
593 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand);
594 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom);
595 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom);
596 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom);
597 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom);
598 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom);
599 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
600 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
601
602 // We want to custom lower some of our intrinsics.
603 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
604
605 // To handle counter-based loop conditions.
606 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom);
607
608 setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom);
609 setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom);
610 setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom);
611 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
612
613 // Comparisons that require checking two conditions.
614 if (Subtarget.hasSPE()) {
615 setCondCodeAction(ISD::SETO, MVT::f32, Expand);
616 setCondCodeAction(ISD::SETO, MVT::f64, Expand);
617 setCondCodeAction(ISD::SETUO, MVT::f32, Expand);
618 setCondCodeAction(ISD::SETUO, MVT::f64, Expand);
619 }
620 setCondCodeAction(ISD::SETULT, MVT::f32, Expand);
621 setCondCodeAction(ISD::SETULT, MVT::f64, Expand);
622 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand);
623 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand);
624 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand);
625 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand);
626 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand);
627 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand);
628 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand);
629 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand);
630 setCondCodeAction(ISD::SETONE, MVT::f32, Expand);
631 setCondCodeAction(ISD::SETONE, MVT::f64, Expand);
632
633 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
634 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
635
636 if (Subtarget.has64BitSupport()) {
637 // They also have instructions for converting between i64 and fp.
638 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
639 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Expand);
640 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
641 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand);
642 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
643 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand);
644 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
645 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
646 // This is just the low 32 bits of a (signed) fp->i64 conversion.
647 // We cannot do this with Promote because i64 is not a legal type.
648 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
649 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
650
651 if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) {
652 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
653 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
654 }
655 } else {
656 // PowerPC does not have FP_TO_UINT on 32-bit implementations.
657 if (Subtarget.hasSPE()) {
658 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Legal);
659 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
660 } else {
661 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Expand);
662 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand);
663 }
664 }
665
666 // With the instructions enabled under FPCVT, we can do everything.
667 if (Subtarget.hasFPCVT()) {
668 if (Subtarget.has64BitSupport()) {
669 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
670 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
671 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
672 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
673 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
674 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
675 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
676 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
677 }
678
679 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
680 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
681 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
682 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
683 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
684 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
685 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
686 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
687 }
688
689 if (Subtarget.use64BitRegs()) {
690 // 64-bit PowerPC implementations can support i64 types directly
691 addRegisterClass(MVT::i64, &PPC::G8RCRegClass);
692 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
693 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
694 // 64-bit PowerPC wants to expand i128 shifts itself.
695 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
696 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
697 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
698 } else {
699 // 32-bit PowerPC wants to expand i64 shifts itself.
700 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
701 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
702 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
703 }
704
705 // PowerPC has better expansions for funnel shifts than the generic
706 // TargetLowering::expandFunnelShift.
707 if (Subtarget.has64BitSupport()) {
708 setOperationAction(ISD::FSHL, MVT::i64, Custom);
709 setOperationAction(ISD::FSHR, MVT::i64, Custom);
710 }
711 setOperationAction(ISD::FSHL, MVT::i32, Custom);
712 setOperationAction(ISD::FSHR, MVT::i32, Custom);
713
714 if (Subtarget.hasVSX()) {
715 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal);
716 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
717 setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
718 setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal);
719 }
720
721 if (Subtarget.hasAltivec()) {
722 for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
723 setOperationAction(ISD::SADDSAT, VT, Legal);
724 setOperationAction(ISD::SSUBSAT, VT, Legal);
725 setOperationAction(ISD::UADDSAT, VT, Legal);
726 setOperationAction(ISD::USUBSAT, VT, Legal);
727 }
728 // First set operation action for all vector types to expand. Then we
729 // will selectively turn on ones that can be effectively codegen'd.
730 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
731 // add/sub are legal for all supported vector VT's.
732 setOperationAction(ISD::ADD, VT, Legal);
733 setOperationAction(ISD::SUB, VT, Legal);
734
735 // For v2i64, these are only valid with P8Vector. This is corrected after
736 // the loop.
737 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) {
738 setOperationAction(ISD::SMAX, VT, Legal);
739 setOperationAction(ISD::SMIN, VT, Legal);
740 setOperationAction(ISD::UMAX, VT, Legal);
741 setOperationAction(ISD::UMIN, VT, Legal);
742 }
743 else {
744 setOperationAction(ISD::SMAX, VT, Expand);
745 setOperationAction(ISD::SMIN, VT, Expand);
746 setOperationAction(ISD::UMAX, VT, Expand);
747 setOperationAction(ISD::UMIN, VT, Expand);
748 }
749
750 if (Subtarget.hasVSX()) {
751 setOperationAction(ISD::FMAXNUM, VT, Legal);
752 setOperationAction(ISD::FMINNUM, VT, Legal);
753 }
754
755 // Vector instructions introduced in P8
756 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
757 setOperationAction(ISD::CTPOP, VT, Legal);
758 setOperationAction(ISD::CTLZ, VT, Legal);
759 }
760 else {
761 setOperationAction(ISD::CTPOP, VT, Expand);
762 setOperationAction(ISD::CTLZ, VT, Expand);
763 }
764
765 // Vector instructions introduced in P9
766 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
767 setOperationAction(ISD::CTTZ, VT, Legal);
768 else
769 setOperationAction(ISD::CTTZ, VT, Expand);
770
771 // We promote all shuffles to v16i8.
772 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote);
773 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8);
774
775 // We promote all non-typed operations to v4i32.
776 setOperationAction(ISD::AND , VT, Promote);
777 AddPromotedToType (ISD::AND , VT, MVT::v4i32);
778 setOperationAction(ISD::OR , VT, Promote);
779 AddPromotedToType (ISD::OR , VT, MVT::v4i32);
780 setOperationAction(ISD::XOR , VT, Promote);
781 AddPromotedToType (ISD::XOR , VT, MVT::v4i32);
782 setOperationAction(ISD::LOAD , VT, Promote);
783 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32);
784 setOperationAction(ISD::SELECT, VT, Promote);
785 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32);
786 setOperationAction(ISD::VSELECT, VT, Legal);
787 setOperationAction(ISD::SELECT_CC, VT, Promote);
788 AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32);
789 setOperationAction(ISD::STORE, VT, Promote);
790 AddPromotedToType (ISD::STORE, VT, MVT::v4i32);
791
792 // No other operations are legal.
793 setOperationAction(ISD::MUL , VT, Expand);
794 setOperationAction(ISD::SDIV, VT, Expand);
795 setOperationAction(ISD::SREM, VT, Expand);
796 setOperationAction(ISD::UDIV, VT, Expand);
797 setOperationAction(ISD::UREM, VT, Expand);
798 setOperationAction(ISD::FDIV, VT, Expand);
799 setOperationAction(ISD::FREM, VT, Expand);
800 setOperationAction(ISD::FNEG, VT, Expand);
801 setOperationAction(ISD::FSQRT, VT, Expand);
802 setOperationAction(ISD::FLOG, VT, Expand);
803 setOperationAction(ISD::FLOG10, VT, Expand);
804 setOperationAction(ISD::FLOG2, VT, Expand);
805 setOperationAction(ISD::FEXP, VT, Expand);
806 setOperationAction(ISD::FEXP2, VT, Expand);
807 setOperationAction(ISD::FSIN, VT, Expand);
808 setOperationAction(ISD::FCOS, VT, Expand);
809 setOperationAction(ISD::FABS, VT, Expand);
810 setOperationAction(ISD::FFLOOR, VT, Expand);
811 setOperationAction(ISD::FCEIL, VT, Expand);
812 setOperationAction(ISD::FTRUNC, VT, Expand);
813 setOperationAction(ISD::FRINT, VT, Expand);
814 setOperationAction(ISD::FNEARBYINT, VT, Expand);
815 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand);
816 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
817 setOperationAction(ISD::BUILD_VECTOR, VT, Expand);
818 setOperationAction(ISD::MULHU, VT, Expand);
819 setOperationAction(ISD::MULHS, VT, Expand);
820 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
821 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
822 setOperationAction(ISD::UDIVREM, VT, Expand);
823 setOperationAction(ISD::SDIVREM, VT, Expand);
824 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
825 setOperationAction(ISD::FPOW, VT, Expand);
826 setOperationAction(ISD::BSWAP, VT, Expand);
827 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
828 setOperationAction(ISD::ROTL, VT, Expand);
829 setOperationAction(ISD::ROTR, VT, Expand);
830
831 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
832 setTruncStoreAction(VT, InnerVT, Expand);
833 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
834 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
835 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
836 }
837 }
838 setOperationAction(ISD::SELECT_CC, MVT::v4i32, Expand);
839 if (!Subtarget.hasP8Vector()) {
840 setOperationAction(ISD::SMAX, MVT::v2i64, Expand);
841 setOperationAction(ISD::SMIN, MVT::v2i64, Expand);
842 setOperationAction(ISD::UMAX, MVT::v2i64, Expand);
843 setOperationAction(ISD::UMIN, MVT::v2i64, Expand);
844 }
845
846 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
847 // with merges, splats, etc.
848 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom);
849
850 // Vector truncates to sub-word integer that fit in an Altivec/VSX register
851 // are cheap, so handle them before they get expanded to scalar.
852 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
853 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
854 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
855 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
856 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
857
858 setOperationAction(ISD::AND , MVT::v4i32, Legal);
859 setOperationAction(ISD::OR , MVT::v4i32, Legal);
860 setOperationAction(ISD::XOR , MVT::v4i32, Legal);
861 setOperationAction(ISD::LOAD , MVT::v4i32, Legal);
862 setOperationAction(ISD::SELECT, MVT::v4i32,
863 Subtarget.useCRBits() ? Legal : Expand);
864 setOperationAction(ISD::STORE , MVT::v4i32, Legal);
865 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal);
866 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal);
867 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal);
868 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal);
869 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
870 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
871 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
872 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
873 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
874 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
875 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
876 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
877
878 // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8.
879 setOperationAction(ISD::ROTL, MVT::v1i128, Custom);
880 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w).
881 if (Subtarget.hasAltivec())
882 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8})
883 setOperationAction(ISD::ROTL, VT, Legal);
884 // With hasP8Altivec set, we can lower ISD::ROTL to vrld.
885 if (Subtarget.hasP8Altivec())
886 setOperationAction(ISD::ROTL, MVT::v2i64, Legal);
887
888 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass);
889 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass);
890 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass);
891 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass);
892
893 setOperationAction(ISD::MUL, MVT::v4f32, Legal);
894 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
895
896 if (Subtarget.hasVSX()) {
897 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
898 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
899 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
900 }
901
902 if (Subtarget.hasP8Altivec())
903 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
904 else
905 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
906
907 if (Subtarget.isISA3_1()) {
908 setOperationAction(ISD::MUL, MVT::v2i64, Legal);
909 setOperationAction(ISD::MULHS, MVT::v2i64, Legal);
910 setOperationAction(ISD::MULHU, MVT::v2i64, Legal);
911 setOperationAction(ISD::MULHS, MVT::v4i32, Legal);
912 setOperationAction(ISD::MULHU, MVT::v4i32, Legal);
913 setOperationAction(ISD::UDIV, MVT::v2i64, Legal);
914 setOperationAction(ISD::SDIV, MVT::v2i64, Legal);
915 setOperationAction(ISD::UDIV, MVT::v4i32, Legal);
916 setOperationAction(ISD::SDIV, MVT::v4i32, Legal);
917 setOperationAction(ISD::UREM, MVT::v2i64, Legal);
918 setOperationAction(ISD::SREM, MVT::v2i64, Legal);
919 setOperationAction(ISD::UREM, MVT::v4i32, Legal);
920 setOperationAction(ISD::SREM, MVT::v4i32, Legal);
921 setOperationAction(ISD::UREM, MVT::v1i128, Legal);
922 setOperationAction(ISD::SREM, MVT::v1i128, Legal);
923 setOperationAction(ISD::UDIV, MVT::v1i128, Legal);
924 setOperationAction(ISD::SDIV, MVT::v1i128, Legal);
925 setOperationAction(ISD::ROTL, MVT::v1i128, Legal);
926 }
927
928 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
929 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
930
931 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
932 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom);
933
934 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom);
935 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom);
936 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom);
937 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
938
939 // Altivec does not contain unordered floating-point compare instructions
940 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand);
941 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand);
942 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand);
943 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand);
944
945 if (Subtarget.hasVSX()) {
946 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
947 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal);
948 if (Subtarget.hasP8Vector()) {
949 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
950 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal);
951 }
952 if (Subtarget.hasDirectMove() && isPPC64) {
953 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal);
954 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal);
955 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal);
956 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal);
957 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal);
958 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal);
959 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal);
960 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal);
961 }
962 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal);
963
964 // The nearbyint variants are not allowed to raise the inexact exception
965 // so we can only code-gen them with unsafe math.
966 if (TM.Options.UnsafeFPMath) {
967 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
968 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
969 }
970
971 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
972 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
973 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
974 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
975 setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
976 setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
977 setOperationAction(ISD::FROUND, MVT::f64, Legal);
978 setOperationAction(ISD::FRINT, MVT::f64, Legal);
979
980 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
981 setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
982 setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
983 setOperationAction(ISD::FROUND, MVT::f32, Legal);
984 setOperationAction(ISD::FRINT, MVT::f32, Legal);
985
986 setOperationAction(ISD::MUL, MVT::v2f64, Legal);
987 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
988
989 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
990 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
991
992 // Share the Altivec comparison restrictions.
993 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand);
994 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand);
995 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand);
996 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand);
997
998 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
999 setOperationAction(ISD::STORE, MVT::v2f64, Legal);
1000
1001 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal);
1002
1003 if (Subtarget.hasP8Vector())
1004 addRegisterClass(MVT::f32, &PPC::VSSRCRegClass);
1005
1006 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass);
1007
1008 addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass);
1009 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass);
1010 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass);
1011
1012 if (Subtarget.hasP8Altivec()) {
1013 setOperationAction(ISD::SHL, MVT::v2i64, Legal);
1014 setOperationAction(ISD::SRA, MVT::v2i64, Legal);
1015 setOperationAction(ISD::SRL, MVT::v2i64, Legal);
1016
1017 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1018 // SRL, but not for SRA because of the instructions available:
1019 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth
1020 // doing
1021 setOperationAction(ISD::SHL, MVT::v1i128, Expand);
1022 setOperationAction(ISD::SRL, MVT::v1i128, Expand);
1023 setOperationAction(ISD::SRA, MVT::v1i128, Expand);
1024
1025 setOperationAction(ISD::SETCC, MVT::v2i64, Legal);
1026 }
1027 else {
1028 setOperationAction(ISD::SHL, MVT::v2i64, Expand);
1029 setOperationAction(ISD::SRA, MVT::v2i64, Expand);
1030 setOperationAction(ISD::SRL, MVT::v2i64, Expand);
1031
1032 setOperationAction(ISD::SETCC, MVT::v2i64, Custom);
1033
1034 // VSX v2i64 only supports non-arithmetic operations.
1035 setOperationAction(ISD::ADD, MVT::v2i64, Expand);
1036 setOperationAction(ISD::SUB, MVT::v2i64, Expand);
1037 }
1038
1039 if (Subtarget.isISA3_1())
1040 setOperationAction(ISD::SETCC, MVT::v1i128, Legal);
1041 else
1042 setOperationAction(ISD::SETCC, MVT::v1i128, Expand);
1043
1044 setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
1045 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64);
1046 setOperationAction(ISD::STORE, MVT::v2i64, Promote);
1047 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64);
1048
1049 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal);
1050
1051 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i64, Legal);
1052 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i64, Legal);
1053 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i64, Legal);
1054 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i64, Legal);
1055 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
1056 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
1057 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
1058 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
1059
1060 // Custom handling for partial vectors of integers converted to
1061 // floating point. We already have optimal handling for v2i32 through
1062 // the DAG combine, so those aren't necessary.
1063 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i8, Custom);
1064 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i8, Custom);
1065 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i16, Custom);
1066 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i16, Custom);
1067 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i8, Custom);
1068 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i8, Custom);
1069 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i16, Custom);
1070 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i16, Custom);
1071 setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom);
1072 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom);
1073 setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom);
1074 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1075 setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom);
1076 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom);
1077 setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom);
1078 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
1079
1080 setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
1081 setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
1082 setOperationAction(ISD::FABS, MVT::v4f32, Legal);
1083 setOperationAction(ISD::FABS, MVT::v2f64, Legal);
1084 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal);
1085 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal);
1086
1087 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
1088 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
1089
1090 // Handle constrained floating-point operations of vector.
1091 // The predictor is `hasVSX` because altivec instruction has
1092 // no exception but VSX vector instruction has.
1093 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
1094 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
1095 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
1096 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
1097 setOperationAction(ISD::STRICT_FMA, MVT::v4f32, Legal);
1098 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
1099 setOperationAction(ISD::STRICT_FMAXNUM, MVT::v4f32, Legal);
1100 setOperationAction(ISD::STRICT_FMINNUM, MVT::v4f32, Legal);
1101 setOperationAction(ISD::STRICT_FRINT, MVT::v4f32, Legal);
1102 setOperationAction(ISD::STRICT_FFLOOR, MVT::v4f32, Legal);
1103 setOperationAction(ISD::STRICT_FCEIL, MVT::v4f32, Legal);
1104 setOperationAction(ISD::STRICT_FTRUNC, MVT::v4f32, Legal);
1105 setOperationAction(ISD::STRICT_FROUND, MVT::v4f32, Legal);
1106
1107 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
1108 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
1109 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
1110 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
1111 setOperationAction(ISD::STRICT_FMA, MVT::v2f64, Legal);
1112 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
1113 setOperationAction(ISD::STRICT_FMAXNUM, MVT::v2f64, Legal);
1114 setOperationAction(ISD::STRICT_FMINNUM, MVT::v2f64, Legal);
1115 setOperationAction(ISD::STRICT_FRINT, MVT::v2f64, Legal);
1116 setOperationAction(ISD::STRICT_FFLOOR, MVT::v2f64, Legal);
1117 setOperationAction(ISD::STRICT_FCEIL, MVT::v2f64, Legal);
1118 setOperationAction(ISD::STRICT_FTRUNC, MVT::v2f64, Legal);
1119 setOperationAction(ISD::STRICT_FROUND, MVT::v2f64, Legal);
1120
1121 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass);
1122 addRegisterClass(MVT::f128, &PPC::VRRCRegClass);
1123
1124 for (MVT FPT : MVT::fp_valuetypes())
1125 setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand);
1126
1127 // Expand the SELECT to SELECT_CC
1128 setOperationAction(ISD::SELECT, MVT::f128, Expand);
1129
1130 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
1131 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
1132
1133 // No implementation for these ops for PowerPC.
1134 setOperationAction(ISD::FSIN, MVT::f128, Expand);
1135 setOperationAction(ISD::FCOS, MVT::f128, Expand);
1136 setOperationAction(ISD::FPOW, MVT::f128, Expand);
1137 setOperationAction(ISD::FPOWI, MVT::f128, Expand);
1138 setOperationAction(ISD::FREM, MVT::f128, Expand);
1139 }
1140
1141 if (Subtarget.hasP8Altivec()) {
1142 addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass);
1143 addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass);
1144 }
1145
1146 if (Subtarget.hasP9Vector()) {
1147 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
1148 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
1149
1150 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1151 // SRL, but not for SRA because of the instructions available:
1152 // VS{RL} and VS{RL}O.
1153 setOperationAction(ISD::SHL, MVT::v1i128, Legal);
1154 setOperationAction(ISD::SRL, MVT::v1i128, Legal);
1155 setOperationAction(ISD::SRA, MVT::v1i128, Expand);
1156
1157 setOperationAction(ISD::FADD, MVT::f128, Legal);
1158 setOperationAction(ISD::FSUB, MVT::f128, Legal);
1159 setOperationAction(ISD::FDIV, MVT::f128, Legal);
1160 setOperationAction(ISD::FMUL, MVT::f128, Legal);
1161 setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal);
1162
1163 setOperationAction(ISD::FMA, MVT::f128, Legal);
1164 setCondCodeAction(ISD::SETULT, MVT::f128, Expand);
1165 setCondCodeAction(ISD::SETUGT, MVT::f128, Expand);
1166 setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand);
1167 setCondCodeAction(ISD::SETOGE, MVT::f128, Expand);
1168 setCondCodeAction(ISD::SETOLE, MVT::f128, Expand);
1169 setCondCodeAction(ISD::SETONE, MVT::f128, Expand);
1170
1171 setOperationAction(ISD::FTRUNC, MVT::f128, Legal);
1172 setOperationAction(ISD::FRINT, MVT::f128, Legal);
1173 setOperationAction(ISD::FFLOOR, MVT::f128, Legal);
1174 setOperationAction(ISD::FCEIL, MVT::f128, Legal);
1175 setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal);
1176 setOperationAction(ISD::FROUND, MVT::f128, Legal);
1177
1178 setOperationAction(ISD::FP_ROUND, MVT::f64, Legal);
1179 setOperationAction(ISD::FP_ROUND, MVT::f32, Legal);
1180 setOperationAction(ISD::BITCAST, MVT::i128, Custom);
1181
1182 // Handle constrained floating-point operations of fp128
1183 setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal);
1184 setOperationAction(ISD::STRICT_FSUB, MVT::f128, Legal);
1185 setOperationAction(ISD::STRICT_FMUL, MVT::f128, Legal);
1186 setOperationAction(ISD::STRICT_FDIV, MVT::f128, Legal);
1187 setOperationAction(ISD::STRICT_FMA, MVT::f128, Legal);
1188 setOperationAction(ISD::STRICT_FSQRT, MVT::f128, Legal);
1189 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Legal);
1190 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
1191 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
1192 setOperationAction(ISD::STRICT_FRINT, MVT::f128, Legal);
1193 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f128, Legal);
1194 setOperationAction(ISD::STRICT_FFLOOR, MVT::f128, Legal);
1195 setOperationAction(ISD::STRICT_FCEIL, MVT::f128, Legal);
1196 setOperationAction(ISD::STRICT_FTRUNC, MVT::f128, Legal);
1197 setOperationAction(ISD::STRICT_FROUND, MVT::f128, Legal);
1198 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
1199 setOperationAction(ISD::BSWAP, MVT::v8i16, Legal);
1200 setOperationAction(ISD::BSWAP, MVT::v4i32, Legal);
1201 setOperationAction(ISD::BSWAP, MVT::v2i64, Legal);
1202 setOperationAction(ISD::BSWAP, MVT::v1i128, Legal);
1203 } else if (Subtarget.hasVSX()) {
1204 setOperationAction(ISD::LOAD, MVT::f128, Promote);
1205 setOperationAction(ISD::STORE, MVT::f128, Promote);
1206
1207 AddPromotedToType(ISD::LOAD, MVT::f128, MVT::v4i32);
1208 AddPromotedToType(ISD::STORE, MVT::f128, MVT::v4i32);
1209
1210 // Set FADD/FSUB as libcall to avoid the legalizer to expand the
1211 // fp_to_uint and int_to_fp.
1212 setOperationAction(ISD::FADD, MVT::f128, LibCall);
1213 setOperationAction(ISD::FSUB, MVT::f128, LibCall);
1214
1215 setOperationAction(ISD::FMUL, MVT::f128, Expand);
1216 setOperationAction(ISD::FDIV, MVT::f128, Expand);
1217 setOperationAction(ISD::FNEG, MVT::f128, Expand);
1218 setOperationAction(ISD::FABS, MVT::f128, Expand);
1219 setOperationAction(ISD::FSQRT, MVT::f128, Expand);
1220 setOperationAction(ISD::FMA, MVT::f128, Expand);
1221 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
1222
1223 // Expand the fp_extend if the target type is fp128.
1224 setOperationAction(ISD::FP_EXTEND, MVT::f128, Expand);
1225 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Expand);
1226
1227 // Expand the fp_round if the source type is fp128.
1228 for (MVT VT : {MVT::f32, MVT::f64}) {
1229 setOperationAction(ISD::FP_ROUND, VT, Custom);
1230 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
1231 }
1232
1233 setOperationAction(ISD::SETCC, MVT::f128, Custom);
1234 setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
1235 setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
1236 setOperationAction(ISD::BR_CC, MVT::f128, Expand);
1237
1238 // Lower following f128 select_cc pattern:
1239 // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE
1240 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
1241
1242 // We need to handle f128 SELECT_CC with integer result type.
1243 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
1244 setOperationAction(ISD::SELECT_CC, MVT::i64, isPPC64 ? Custom : Expand);
1245 }
1246
1247 if (Subtarget.hasP9Altivec()) {
1248 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
1249 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1250
1251 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal);
1252 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal);
1253 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal);
1254 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal);
1255 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);
1256 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
1257 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal);
1258 }
1259
1260 if (Subtarget.isISA3_1())
1261 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom);
1262 }
1263
1264 if (Subtarget.pairedVectorMemops()) {
1265 addRegisterClass(MVT::v256i1, &PPC::VSRpRCRegClass);
1266 setOperationAction(ISD::LOAD, MVT::v256i1, Custom);
1267 setOperationAction(ISD::STORE, MVT::v256i1, Custom);
1268 }
1269 if (Subtarget.hasMMA()) {
1270 addRegisterClass(MVT::v512i1, &PPC::UACCRCRegClass);
1271 setOperationAction(ISD::LOAD, MVT::v512i1, Custom);
1272 setOperationAction(ISD::STORE, MVT::v512i1, Custom);
1273 setOperationAction(ISD::BUILD_VECTOR, MVT::v512i1, Custom);
1274 }
1275
1276 if (Subtarget.has64BitSupport())
1277 setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
1278
1279 if (Subtarget.isISA3_1())
1280 setOperationAction(ISD::SRA, MVT::v1i128, Legal);
1281
1282 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom);
1283
1284 if (!isPPC64) {
1285 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand);
1286 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
1287 }
1288
1289 if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics())
1290 setMaxAtomicSizeInBitsSupported(128);
1291
1292 setBooleanContents(ZeroOrOneBooleanContent);
1293
1294 if (Subtarget.hasAltivec()) {
1295 // Altivec instructions set fields to all zeros or all ones.
1296 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
1297 }
1298
1299 if (!isPPC64) {
1300 // These libcalls are not available in 32-bit.
1301 setLibcallName(RTLIB::SHL_I128, nullptr);
1302 setLibcallName(RTLIB::SRL_I128, nullptr);
1303 setLibcallName(RTLIB::SRA_I128, nullptr);
1304 }
1305
1306 if (!isPPC64)
1307 setMaxAtomicSizeInBitsSupported(32);
1308
1309 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
1310
1311 // We have target-specific dag combine patterns for the following nodes:
1312 setTargetDAGCombine(ISD::ADD);
1313 setTargetDAGCombine(ISD::SHL);
1314 setTargetDAGCombine(ISD::SRA);
1315 setTargetDAGCombine(ISD::SRL);
1316 setTargetDAGCombine(ISD::MUL);
1317 setTargetDAGCombine(ISD::FMA);
1318 setTargetDAGCombine(ISD::SINT_TO_FP);
1319 setTargetDAGCombine(ISD::BUILD_VECTOR);
1320 if (Subtarget.hasFPCVT())
1321 setTargetDAGCombine(ISD::UINT_TO_FP);
1322 setTargetDAGCombine(ISD::LOAD);
1323 setTargetDAGCombine(ISD::STORE);
1324 setTargetDAGCombine(ISD::BR_CC);
1325 if (Subtarget.useCRBits())
1326 setTargetDAGCombine(ISD::BRCOND);
1327 setTargetDAGCombine(ISD::BSWAP);
1328 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
1329 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
1330 setTargetDAGCombine(ISD::INTRINSIC_VOID);
1331
1332 setTargetDAGCombine(ISD::SIGN_EXTEND);
1333 setTargetDAGCombine(ISD::ZERO_EXTEND);
1334 setTargetDAGCombine(ISD::ANY_EXTEND);
1335
1336 setTargetDAGCombine(ISD::TRUNCATE);
1337 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
1338
1339
1340 if (Subtarget.useCRBits()) {
1341 setTargetDAGCombine(ISD::TRUNCATE);
1342 setTargetDAGCombine(ISD::SETCC);
1343 setTargetDAGCombine(ISD::SELECT_CC);
1344 }
1345
1346 if (Subtarget.hasP9Altivec()) {
1347 setTargetDAGCombine(ISD::ABS);
1348 setTargetDAGCombine(ISD::VSELECT);
1349 }
1350
1351 setLibcallName(RTLIB::LOG_F128, "logf128");
1352 setLibcallName(RTLIB::LOG2_F128, "log2f128");
1353 setLibcallName(RTLIB::LOG10_F128, "log10f128");
1354 setLibcallName(RTLIB::EXP_F128, "expf128");
1355 setLibcallName(RTLIB::EXP2_F128, "exp2f128");
1356 setLibcallName(RTLIB::SIN_F128, "sinf128");
1357 setLibcallName(RTLIB::COS_F128, "cosf128");
1358 setLibcallName(RTLIB::POW_F128, "powf128");
1359 setLibcallName(RTLIB::FMIN_F128, "fminf128");
1360 setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
1361 setLibcallName(RTLIB::REM_F128, "fmodf128");
1362 setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
1363 setLibcallName(RTLIB::CEIL_F128, "ceilf128");
1364 setLibcallName(RTLIB::FLOOR_F128, "floorf128");
1365 setLibcallName(RTLIB::TRUNC_F128, "truncf128");
1366 setLibcallName(RTLIB::ROUND_F128, "roundf128");
1367 setLibcallName(RTLIB::LROUND_F128, "lroundf128");
1368 setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
1369 setLibcallName(RTLIB::RINT_F128, "rintf128");
1370 setLibcallName(RTLIB::LRINT_F128, "lrintf128");
1371 setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
1372 setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
1373 setLibcallName(RTLIB::FMA_F128, "fmaf128");
1374
1375 // With 32 condition bits, we don't need to sink (and duplicate) compares
1376 // aggressively in CodeGenPrep.
1377 if (Subtarget.useCRBits()) {
1378 setHasMultipleConditionRegisters();
1379 setJumpIsExpensive();
1380 }
1381
1382 setMinFunctionAlignment(Align(4));
1383
1384 switch (Subtarget.getCPUDirective()) {
1385 default: break;
1386 case PPC::DIR_970:
1387 case PPC::DIR_A2:
1388 case PPC::DIR_E500:
1389 case PPC::DIR_E500mc:
1390 case PPC::DIR_E5500:
1391 case PPC::DIR_PWR4:
1392 case PPC::DIR_PWR5:
1393 case PPC::DIR_PWR5X:
1394 case PPC::DIR_PWR6:
1395 case PPC::DIR_PWR6X:
1396 case PPC::DIR_PWR7:
1397 case PPC::DIR_PWR8:
1398 case PPC::DIR_PWR9:
1399 case PPC::DIR_PWR10:
1400 case PPC::DIR_PWR_FUTURE:
1401 setPrefLoopAlignment(Align(16));
1402 setPrefFunctionAlignment(Align(16));
1403 break;
1404 }
1405
1406 if (Subtarget.enableMachineScheduler())
1407 setSchedulingPreference(Sched::Source);
1408 else
1409 setSchedulingPreference(Sched::Hybrid);
1410
1411 computeRegisterProperties(STI.getRegisterInfo());
1412
1413 // The Freescale cores do better with aggressive inlining of memcpy and
1414 // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
1415 if (Subtarget.getCPUDirective() == PPC::DIR_E500mc ||
1416 Subtarget.getCPUDirective() == PPC::DIR_E5500) {
1417 MaxStoresPerMemset = 32;
1418 MaxStoresPerMemsetOptSize = 16;
1419 MaxStoresPerMemcpy = 32;
1420 MaxStoresPerMemcpyOptSize = 8;
1421 MaxStoresPerMemmove = 32;
1422 MaxStoresPerMemmoveOptSize = 8;
1423 } else if (Subtarget.getCPUDirective() == PPC::DIR_A2) {
1424 // The A2 also benefits from (very) aggressive inlining of memcpy and
1425 // friends. The overhead of a the function call, even when warm, can be
1426 // over one hundred cycles.
1427 MaxStoresPerMemset = 128;
1428 MaxStoresPerMemcpy = 128;
1429 MaxStoresPerMemmove = 128;
1430 MaxLoadsPerMemcmp = 128;
1431 } else {
1432 MaxLoadsPerMemcmp = 8;
1433 MaxLoadsPerMemcmpOptSize = 4;
1434 }
1435
1436 IsStrictFPEnabled = true;
1437
1438 // Let the subtarget (CPU) decide if a predictable select is more expensive
1439 // than the corresponding branch. This information is used in CGP to decide
1440 // when to convert selects into branches.
1441 PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive();
1442}
1443
1444// *********************************** NOTE ************************************
1445// For selecting load and store instructions, the addressing modes are defined
1446// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD
1447// patterns to match the load the store instructions.
1448//
1449// The TD definitions for the addressing modes correspond to their respective
1450// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely
1451// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the
1452// address mode flags of a particular node. Afterwards, the computed address
1453// flags are passed into getAddrModeForFlags() in order to retrieve the optimal
1454// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement
1455// accordingly, based on the preferred addressing mode.
1456//
1457// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode.
1458// MemOpFlags contains all the possible flags that can be used to compute the
1459// optimal addressing mode for load and store instructions.
1460// AddrMode contains all the possible load and store addressing modes available
1461// on Power (such as DForm, DSForm, DQForm, XForm, etc.)
1462//
1463// When adding new load and store instructions, it is possible that new address
1464// flags may need to be added into MemOpFlags, and a new addressing mode will
1465// need to be added to AddrMode. An entry of the new addressing mode (consisting
1466// of the minimal and main distinguishing address flags for the new load/store
1467// instructions) will need to be added into initializeAddrModeMap() below.
1468// Finally, when adding new addressing modes, the getAddrModeForFlags() will
1469// need to be updated to account for selecting the optimal addressing mode.
1470// *****************************************************************************
1471/// Initialize the map that relates the different addressing modes of the load
1472/// and store instructions to a set of flags. This ensures the load/store
1473/// instruction is correctly matched during instruction selection.
1474void PPCTargetLowering::initializeAddrModeMap() {
1475 AddrModesMap[PPC::AM_DForm] = {
1476 // LWZ, STW
1477 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt,
1478 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt,
1479 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1480 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1481 // LBZ, LHZ, STB, STH
1482 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1483 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1484 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1485 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1486 // LHA
1487 PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1488 PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1489 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1490 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1491 // LFS, LFD, STFS, STFD
1492 PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1493 PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1494 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1495 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1496 };
1497 AddrModesMap[PPC::AM_DSForm] = {
1498 // LWA
1499 PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt,
1500 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1501 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1502 // LD, STD
1503 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt,
1504 PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt,
1505 PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt,
1506 // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64
1507 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1508 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1509 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1510 };
1511 AddrModesMap[PPC::AM_DQForm] = {
1512 // LXV, STXV
1513 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1514 PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1515 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1516 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10,
1517 PPC::MOF_NotAddNorCst | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10,
1518 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10,
1519 };
1520}
1521
1522/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
1523/// the desired ByVal argument alignment.
1524static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
1525 if (MaxAlign == MaxMaxAlign)
1526 return;
1527 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1528 if (MaxMaxAlign >= 32 &&
1529 VTy->getPrimitiveSizeInBits().getFixedSize() >= 256)
1530 MaxAlign = Align(32);
1531 else if (VTy->getPrimitiveSizeInBits().getFixedSize() >= 128 &&
1532 MaxAlign < 16)
1533 MaxAlign = Align(16);
1534 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
1535 Align EltAlign;
1536 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign);
1537 if (EltAlign > MaxAlign)
1538 MaxAlign = EltAlign;
1539 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
1540 for (auto *EltTy : STy->elements()) {
1541 Align EltAlign;
1542 getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign);
1543 if (EltAlign > MaxAlign)
1544 MaxAlign = EltAlign;
1545 if (MaxAlign == MaxMaxAlign)
1546 break;
1547 }
1548 }
1549}
1550
1551/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1552/// function arguments in the caller parameter area.
1553unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty,
1554 const DataLayout &DL) const {
1555 // 16byte and wider vectors are passed on 16byte boundary.
1556 // The rest is 8 on PPC64 and 4 on PPC32 boundary.
1557 Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
1558 if (Subtarget.hasAltivec())
1559 getMaxByValAlign(Ty, Alignment, Align(16));
1560 return Alignment.value();
1561}
1562
1563bool PPCTargetLowering::useSoftFloat() const {
1564 return Subtarget.useSoftFloat();
1565}
1566
1567bool PPCTargetLowering::hasSPE() const {
1568 return Subtarget.hasSPE();
1569}
1570
1571bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
1572 return VT.isScalarInteger();
1573}
1574
1575const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
1576 switch ((PPCISD::NodeType)Opcode) {
1577 case PPCISD::FIRST_NUMBER: break;
1578 case PPCISD::FSEL: return "PPCISD::FSEL";
1579 case PPCISD::XSMAXCDP: return "PPCISD::XSMAXCDP";
1580 case PPCISD::XSMINCDP: return "PPCISD::XSMINCDP";
1581 case PPCISD::FCFID: return "PPCISD::FCFID";
1582 case PPCISD::FCFIDU: return "PPCISD::FCFIDU";
1583 case PPCISD::FCFIDS: return "PPCISD::FCFIDS";
1584 case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS";
1585 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ";
1586 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ";
1587 case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ";
1588 case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ";
1589 case PPCISD::FP_TO_UINT_IN_VSR:
1590 return "PPCISD::FP_TO_UINT_IN_VSR,";
1591 case PPCISD::FP_TO_SINT_IN_VSR:
1592 return "PPCISD::FP_TO_SINT_IN_VSR";
1593 case PPCISD::FRE: return "PPCISD::FRE";
1594 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE";
1595 case PPCISD::FTSQRT:
1596 return "PPCISD::FTSQRT";
1597 case PPCISD::FSQRT:
1598 return "PPCISD::FSQRT";
1599 case PPCISD::STFIWX: return "PPCISD::STFIWX";
1600 case PPCISD::VPERM: return "PPCISD::VPERM";
1601 case PPCISD::XXSPLT: return "PPCISD::XXSPLT";
1602 case PPCISD::XXSPLTI_SP_TO_DP:
1603 return "PPCISD::XXSPLTI_SP_TO_DP";
1604 case PPCISD::XXSPLTI32DX:
1605 return "PPCISD::XXSPLTI32DX";
1606 case PPCISD::VECINSERT: return "PPCISD::VECINSERT";
1607 case PPCISD::XXPERMDI: return "PPCISD::XXPERMDI";
1608 case PPCISD::VECSHL: return "PPCISD::VECSHL";
1609 case PPCISD::CMPB: return "PPCISD::CMPB";
1610 case PPCISD::Hi: return "PPCISD::Hi";
1611 case PPCISD::Lo: return "PPCISD::Lo";
1612 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY";
1613 case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8";
1614 case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16";
1615 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC";
1616 case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET";
1617 case PPCISD::PROBED_ALLOCA: return "PPCISD::PROBED_ALLOCA";
1618 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg";
1619 case PPCISD::SRL: return "PPCISD::SRL";
1620 case PPCISD::SRA: return "PPCISD::SRA";
1621 case PPCISD::SHL: return "PPCISD::SHL";
1622 case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE";
1623 case PPCISD::CALL: return "PPCISD::CALL";
1624 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP";
1625 case PPCISD::CALL_NOTOC: return "PPCISD::CALL_NOTOC";
1626 case PPCISD::MTCTR: return "PPCISD::MTCTR";
1627 case PPCISD::BCTRL: return "PPCISD::BCTRL";
1628 case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC";
1629 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG";
1630 case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE";
1631 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP";
1632 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP";
1633 case PPCISD::MFOCRF: return "PPCISD::MFOCRF";
1634 case PPCISD::MFVSR: return "PPCISD::MFVSR";
1635 case PPCISD::MTVSRA: return "PPCISD::MTVSRA";
1636 case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ";
1637 case PPCISD::SINT_VEC_TO_FP: return "PPCISD::SINT_VEC_TO_FP";
1638 case PPCISD::UINT_VEC_TO_FP: return "PPCISD::UINT_VEC_TO_FP";
1639 case PPCISD::SCALAR_TO_VECTOR_PERMUTED:
1640 return "PPCISD::SCALAR_TO_VECTOR_PERMUTED";
1641 case PPCISD::ANDI_rec_1_EQ_BIT:
1642 return "PPCISD::ANDI_rec_1_EQ_BIT";
1643 case PPCISD::ANDI_rec_1_GT_BIT:
1644 return "PPCISD::ANDI_rec_1_GT_BIT";
1645 case PPCISD::VCMP: return "PPCISD::VCMP";
1646 case PPCISD::VCMP_rec: return "PPCISD::VCMP_rec";
1647 case PPCISD::LBRX: return "PPCISD::LBRX";
1648 case PPCISD::STBRX: return "PPCISD::STBRX";
1649 case PPCISD::LFIWAX: return "PPCISD::LFIWAX";
1650 case PPCISD::LFIWZX: return "PPCISD::LFIWZX";
1651 case PPCISD::LXSIZX: return "PPCISD::LXSIZX";
1652 case PPCISD::STXSIX: return "PPCISD::STXSIX";
1653 case PPCISD::VEXTS: return "PPCISD::VEXTS";
1654 case PPCISD::LXVD2X: return "PPCISD::LXVD2X";
1655 case PPCISD::STXVD2X: return "PPCISD::STXVD2X";
1656 case PPCISD::LOAD_VEC_BE: return "PPCISD::LOAD_VEC_BE";
1657 case PPCISD::STORE_VEC_BE: return "PPCISD::STORE_VEC_BE";
1658 case PPCISD::ST_VSR_SCAL_INT:
1659 return "PPCISD::ST_VSR_SCAL_INT";
1660 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH";
1661 case PPCISD::BDNZ: return "PPCISD::BDNZ";
1662 case PPCISD::BDZ: return "PPCISD::BDZ";
1663 case PPCISD::MFFS: return "PPCISD::MFFS";
1664 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ";
1665 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN";
1666 case PPCISD::CR6SET: return "PPCISD::CR6SET";
1667 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET";
1668 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT";
1669 case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT";
1670 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA";
1671 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L";
1672 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS";
1673 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA";
1674 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L";
1675 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR";
1676 case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR";
1677 case PPCISD::TLSGD_AIX: return "PPCISD::TLSGD_AIX";
1678 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA";
1679 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L";
1680 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR";
1681 case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR";
1682 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA";
1683 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L";
1684 case PPCISD::PADDI_DTPREL:
1685 return "PPCISD::PADDI_DTPREL";
1686 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT";
1687 case PPCISD::SC: return "PPCISD::SC";
1688 case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB";
1689 case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE";
1690 case PPCISD::RFEBB: return "PPCISD::RFEBB";
1691 case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD";
1692 case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN";
1693 case PPCISD::VABSD: return "PPCISD::VABSD";
1694 case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128";
1695 case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64";
1696 case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE";
1697 case PPCISD::EXTSWSLI: return "PPCISD::EXTSWSLI";
1698 case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH";
1699 case PPCISD::FP_EXTEND_HALF: return "PPCISD::FP_EXTEND_HALF";
1700 case PPCISD::MAT_PCREL_ADDR: return "PPCISD::MAT_PCREL_ADDR";
1701 case PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR:
1702 return "PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR";
1703 case PPCISD::TLS_LOCAL_EXEC_MAT_ADDR:
1704 return "PPCISD::TLS_LOCAL_EXEC_MAT_ADDR";
1705 case PPCISD::ACC_BUILD: return "PPCISD::ACC_BUILD";
1706 case PPCISD::PAIR_BUILD: return "PPCISD::PAIR_BUILD";
1707 case PPCISD::EXTRACT_VSX_REG: return "PPCISD::EXTRACT_VSX_REG";
1708 case PPCISD::XXMFACC: return "PPCISD::XXMFACC";
1709 case PPCISD::LD_SPLAT: return "PPCISD::LD_SPLAT";
1710 case PPCISD::FNMSUB: return "PPCISD::FNMSUB";
1711 case PPCISD::STRICT_FADDRTZ:
1712 return "PPCISD::STRICT_FADDRTZ";
1713 case PPCISD::STRICT_FCTIDZ:
1714 return "PPCISD::STRICT_FCTIDZ";
1715 case PPCISD::STRICT_FCTIWZ:
1716 return "PPCISD::STRICT_FCTIWZ";
1717 case PPCISD::STRICT_FCTIDUZ:
1718 return "PPCISD::STRICT_FCTIDUZ";
1719 case PPCISD::STRICT_FCTIWUZ:
1720 return "PPCISD::STRICT_FCTIWUZ";
1721 case PPCISD::STRICT_FCFID:
1722 return "PPCISD::STRICT_FCFID";
1723 case PPCISD::STRICT_FCFIDU:
1724 return "PPCISD::STRICT_FCFIDU";
1725 case PPCISD::STRICT_FCFIDS:
1726 return "PPCISD::STRICT_FCFIDS";
1727 case PPCISD::STRICT_FCFIDUS:
1728 return "PPCISD::STRICT_FCFIDUS";
1729 case PPCISD::LXVRZX: return "PPCISD::LXVRZX";
1730 }
1731 return nullptr;
1732}
1733
1734EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1735 EVT VT) const {
1736 if (!VT.isVector())
1737 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
1738
1739 return VT.changeVectorElementTypeToInteger();
1740}
1741
1742bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
1743 assert(VT.isFloatingPoint() && "Non-floating-point FMA?")(static_cast <bool> (VT.isFloatingPoint() && "Non-floating-point FMA?"
) ? void (0) : __assert_fail ("VT.isFloatingPoint() && \"Non-floating-point FMA?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 1743, __extension__ __PRETTY_FUNCTION__))
;
1744 return true;
1745}
1746
1747//===----------------------------------------------------------------------===//
1748// Node matching predicates, for use by the tblgen matching code.
1749//===----------------------------------------------------------------------===//
1750
1751/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
1752static bool isFloatingPointZero(SDValue Op) {
1753 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
1754 return CFP->getValueAPF().isZero();
1755 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
1756 // Maybe this has already been legalized into the constant pool?
1757 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1)))
1758 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
1759 return CFP->getValueAPF().isZero();
1760 }
1761 return false;
1762}
1763
1764/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
1765/// true if Op is undef or if it matches the specified value.
1766static bool isConstantOrUndef(int Op, int Val) {
1767 return Op < 0 || Op == Val;
1768}
1769
1770/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
1771/// VPKUHUM instruction.
1772/// The ShuffleKind distinguishes between big-endian operations with
1773/// two different inputs (0), either-endian operations with two identical
1774/// inputs (1), and little-endian operations with two different inputs (2).
1775/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1776bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1777 SelectionDAG &DAG) {
1778 bool IsLE = DAG.getDataLayout().isLittleEndian();
1779 if (ShuffleKind == 0) {
1780 if (IsLE)
1781 return false;
1782 for (unsigned i = 0; i != 16; ++i)
1783 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1))
1784 return false;
1785 } else if (ShuffleKind == 2) {
1786 if (!IsLE)
1787 return false;
1788 for (unsigned i = 0; i != 16; ++i)
1789 if (!isConstantOrUndef(N->getMaskElt(i), i*2))
1790 return false;
1791 } else if (ShuffleKind == 1) {
1792 unsigned j = IsLE ? 0 : 1;
1793 for (unsigned i = 0; i != 8; ++i)
1794 if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) ||
1795 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j))
1796 return false;
1797 }
1798 return true;
1799}
1800
1801/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
1802/// VPKUWUM instruction.
1803/// The ShuffleKind distinguishes between big-endian operations with
1804/// two different inputs (0), either-endian operations with two identical
1805/// inputs (1), and little-endian operations with two different inputs (2).
1806/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1807bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1808 SelectionDAG &DAG) {
1809 bool IsLE = DAG.getDataLayout().isLittleEndian();
1810 if (ShuffleKind == 0) {
1811 if (IsLE)
1812 return false;
1813 for (unsigned i = 0; i != 16; i += 2)
1814 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) ||
1815 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3))
1816 return false;
1817 } else if (ShuffleKind == 2) {
1818 if (!IsLE)
1819 return false;
1820 for (unsigned i = 0; i != 16; i += 2)
1821 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) ||
1822 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1))
1823 return false;
1824 } else if (ShuffleKind == 1) {
1825 unsigned j = IsLE ? 0 : 2;
1826 for (unsigned i = 0; i != 8; i += 2)
1827 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) ||
1828 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) ||
1829 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) ||
1830 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1))
1831 return false;
1832 }
1833 return true;
1834}
1835
1836/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
1837/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
1838/// current subtarget.
1839///
1840/// The ShuffleKind distinguishes between big-endian operations with
1841/// two different inputs (0), either-endian operations with two identical
1842/// inputs (1), and little-endian operations with two different inputs (2).
1843/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1844bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1845 SelectionDAG &DAG) {
1846 const PPCSubtarget& Subtarget =
1847 static_cast<const PPCSubtarget&>(DAG.getSubtarget());
1848 if (!Subtarget.hasP8Vector())
1849 return false;
1850
1851 bool IsLE = DAG.getDataLayout().isLittleEndian();
1852 if (ShuffleKind == 0) {
1853 if (IsLE)
1854 return false;
1855 for (unsigned i = 0; i != 16; i += 4)
1856 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) ||
1857 !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) ||
1858 !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) ||
1859 !isConstantOrUndef(N->getMaskElt(i+3), i*2+7))
1860 return false;
1861 } else if (ShuffleKind == 2) {
1862 if (!IsLE)
1863 return false;
1864 for (unsigned i = 0; i != 16; i += 4)
1865 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) ||
1866 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) ||
1867 !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) ||
1868 !isConstantOrUndef(N->getMaskElt(i+3), i*2+3))
1869 return false;
1870 } else if (ShuffleKind == 1) {
1871 unsigned j = IsLE ? 0 : 4;
1872 for (unsigned i = 0; i != 8; i += 4)
1873 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) ||
1874 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) ||
1875 !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) ||
1876 !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) ||
1877 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) ||
1878 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) ||
1879 !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) ||
1880 !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3))
1881 return false;
1882 }
1883 return true;
1884}
1885
1886/// isVMerge - Common function, used to match vmrg* shuffles.
1887///
1888static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
1889 unsigned LHSStart, unsigned RHSStart) {
1890 if (N->getValueType(0) != MVT::v16i8)
1891 return false;
1892 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&(static_cast <bool> ((UnitSize == 1 || UnitSize == 2 ||
UnitSize == 4) && "Unsupported merge size!") ? void (
0) : __assert_fail ("(UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && \"Unsupported merge size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 1893, __extension__ __PRETTY_FUNCTION__))
1893 "Unsupported merge size!")(static_cast <bool> ((UnitSize == 1 || UnitSize == 2 ||
UnitSize == 4) && "Unsupported merge size!") ? void (
0) : __assert_fail ("(UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && \"Unsupported merge size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 1893, __extension__ __PRETTY_FUNCTION__))
;
1894
1895 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
1896 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
1897 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j),
1898 LHSStart+j+i*UnitSize) ||
1899 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j),
1900 RHSStart+j+i*UnitSize))
1901 return false;
1902 }
1903 return true;
1904}
1905
1906/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
1907/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
1908/// The ShuffleKind distinguishes between big-endian merges with two
1909/// different inputs (0), either-endian merges with two identical inputs (1),
1910/// and little-endian merges with two different inputs (2). For the latter,
1911/// the input operands are swapped (see PPCInstrAltivec.td).
1912bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1913 unsigned ShuffleKind, SelectionDAG &DAG) {
1914 if (DAG.getDataLayout().isLittleEndian()) {
1915 if (ShuffleKind == 1) // unary
1916 return isVMerge(N, UnitSize, 0, 0);
1917 else if (ShuffleKind == 2) // swapped
1918 return isVMerge(N, UnitSize, 0, 16);
1919 else
1920 return false;
1921 } else {
1922 if (ShuffleKind == 1) // unary
1923 return isVMerge(N, UnitSize, 8, 8);
1924 else if (ShuffleKind == 0) // normal
1925 return isVMerge(N, UnitSize, 8, 24);
1926 else
1927 return false;
1928 }
1929}
1930
1931/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
1932/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
1933/// The ShuffleKind distinguishes between big-endian merges with two
1934/// different inputs (0), either-endian merges with two identical inputs (1),
1935/// and little-endian merges with two different inputs (2). For the latter,
1936/// the input operands are swapped (see PPCInstrAltivec.td).
1937bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1938 unsigned ShuffleKind, SelectionDAG &DAG) {
1939 if (DAG.getDataLayout().isLittleEndian()) {
1940 if (ShuffleKind == 1) // unary
1941 return isVMerge(N, UnitSize, 8, 8);
1942 else if (ShuffleKind == 2) // swapped
1943 return isVMerge(N, UnitSize, 8, 24);
1944 else
1945 return false;
1946 } else {
1947 if (ShuffleKind == 1) // unary
1948 return isVMerge(N, UnitSize, 0, 0);
1949 else if (ShuffleKind == 0) // normal
1950 return isVMerge(N, UnitSize, 0, 16);
1951 else
1952 return false;
1953 }
1954}
1955
1956/**
1957 * Common function used to match vmrgew and vmrgow shuffles
1958 *
1959 * The indexOffset determines whether to look for even or odd words in
1960 * the shuffle mask. This is based on the of the endianness of the target
1961 * machine.
1962 * - Little Endian:
1963 * - Use offset of 0 to check for odd elements
1964 * - Use offset of 4 to check for even elements
1965 * - Big Endian:
1966 * - Use offset of 0 to check for even elements
1967 * - Use offset of 4 to check for odd elements
1968 * A detailed description of the vector element ordering for little endian and
1969 * big endian can be found at
1970 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
1971 * Targeting your applications - what little endian and big endian IBM XL C/C++
1972 * compiler differences mean to you
1973 *
1974 * The mask to the shuffle vector instruction specifies the indices of the
1975 * elements from the two input vectors to place in the result. The elements are
1976 * numbered in array-access order, starting with the first vector. These vectors
1977 * are always of type v16i8, thus each vector will contain 16 elements of size
1978 * 8. More info on the shuffle vector can be found in the
1979 * http://llvm.org/docs/LangRef.html#shufflevector-instruction
1980 * Language Reference.
1981 *
1982 * The RHSStartValue indicates whether the same input vectors are used (unary)
1983 * or two different input vectors are used, based on the following:
1984 * - If the instruction uses the same vector for both inputs, the range of the
1985 * indices will be 0 to 15. In this case, the RHSStart value passed should
1986 * be 0.
1987 * - If the instruction has two different vectors then the range of the
1988 * indices will be 0 to 31. In this case, the RHSStart value passed should
1989 * be 16 (indices 0-15 specify elements in the first vector while indices 16
1990 * to 31 specify elements in the second vector).
1991 *
1992 * \param[in] N The shuffle vector SD Node to analyze
1993 * \param[in] IndexOffset Specifies whether to look for even or odd elements
1994 * \param[in] RHSStartValue Specifies the starting index for the righthand input
1995 * vector to the shuffle_vector instruction
1996 * \return true iff this shuffle vector represents an even or odd word merge
1997 */
1998static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
1999 unsigned RHSStartValue) {
2000 if (N->getValueType(0) != MVT::v16i8)
2001 return false;
2002
2003 for (unsigned i = 0; i < 2; ++i)
2004 for (unsigned j = 0; j < 4; ++j)
2005 if (!isConstantOrUndef(N->getMaskElt(i*4+j),
2006 i*RHSStartValue+j+IndexOffset) ||
2007 !isConstantOrUndef(N->getMaskElt(i*4+j+8),
2008 i*RHSStartValue+j+IndexOffset+8))
2009 return false;
2010 return true;
2011}
2012
2013/**
2014 * Determine if the specified shuffle mask is suitable for the vmrgew or
2015 * vmrgow instructions.
2016 *
2017 * \param[in] N The shuffle vector SD Node to analyze
2018 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
2019 * \param[in] ShuffleKind Identify the type of merge:
2020 * - 0 = big-endian merge with two different inputs;
2021 * - 1 = either-endian merge with two identical inputs;
2022 * - 2 = little-endian merge with two different inputs (inputs are swapped for
2023 * little-endian merges).
2024 * \param[in] DAG The current SelectionDAG
2025 * \return true iff this shuffle mask
2026 */
2027bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
2028 unsigned ShuffleKind, SelectionDAG &DAG) {
2029 if (DAG.getDataLayout().isLittleEndian()) {
2030 unsigned indexOffset = CheckEven ? 4 : 0;
2031 if (ShuffleKind == 1) // Unary
2032 return isVMerge(N, indexOffset, 0);
2033 else if (ShuffleKind == 2) // swapped
2034 return isVMerge(N, indexOffset, 16);
2035 else
2036 return false;
2037 }
2038 else {
2039 unsigned indexOffset = CheckEven ? 0 : 4;
2040 if (ShuffleKind == 1) // Unary
2041 return isVMerge(N, indexOffset, 0);
2042 else if (ShuffleKind == 0) // Normal
2043 return isVMerge(N, indexOffset, 16);
2044 else
2045 return false;
2046 }
2047 return false;
2048}
2049
2050/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
2051/// amount, otherwise return -1.
2052/// The ShuffleKind distinguishes between big-endian operations with two
2053/// different inputs (0), either-endian operations with two identical inputs
2054/// (1), and little-endian operations with two different inputs (2). For the
2055/// latter, the input operands are swapped (see PPCInstrAltivec.td).
2056int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
2057 SelectionDAG &DAG) {
2058 if (N->getValueType(0) != MVT::v16i8)
2059 return -1;
2060
2061 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
2062
2063 // Find the first non-undef value in the shuffle mask.
2064 unsigned i;
2065 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i)
2066 /*search*/;
2067
2068 if (i == 16) return -1; // all undef.
2069
2070 // Otherwise, check to see if the rest of the elements are consecutively
2071 // numbered from this value.
2072 unsigned ShiftAmt = SVOp->getMaskElt(i);
2073 if (ShiftAmt < i) return -1;
2074
2075 ShiftAmt -= i;
2076 bool isLE = DAG.getDataLayout().isLittleEndian();
2077
2078 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
2079 // Check the rest of the elements to see if they are consecutive.
2080 for (++i; i != 16; ++i)
2081 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i))
2082 return -1;
2083 } else if (ShuffleKind == 1) {
2084 // Check the rest of the elements to see if they are consecutive.
2085 for (++i; i != 16; ++i)
2086 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15))
2087 return -1;
2088 } else
2089 return -1;
2090
2091 if (isLE)
2092 ShiftAmt = 16 - ShiftAmt;
2093
2094 return ShiftAmt;
2095}
2096
2097/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
2098/// specifies a splat of a single element that is suitable for input to
2099/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
2100bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
2101 assert(N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) &&(static_cast <bool> (N->getValueType(0) == MVT::v16i8
&& isPowerOf2_32(EltSize) && EltSize <= 8
&& "Can only handle 1,2,4,8 byte element sizes") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) && EltSize <= 8 && \"Can only handle 1,2,4,8 byte element sizes\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2102, __extension__ __PRETTY_FUNCTION__))
2102 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes")(static_cast <bool> (N->getValueType(0) == MVT::v16i8
&& isPowerOf2_32(EltSize) && EltSize <= 8
&& "Can only handle 1,2,4,8 byte element sizes") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) && EltSize <= 8 && \"Can only handle 1,2,4,8 byte element sizes\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2102, __extension__ __PRETTY_FUNCTION__))
;
2103
2104 // The consecutive indices need to specify an element, not part of two
2105 // different elements. So abandon ship early if this isn't the case.
2106 if (N->getMaskElt(0) % EltSize != 0)
2107 return false;
2108
2109 // This is a splat operation if each element of the permute is the same, and
2110 // if the value doesn't reference the second vector.
2111 unsigned ElementBase = N->getMaskElt(0);
2112
2113 // FIXME: Handle UNDEF elements too!
2114 if (ElementBase >= 16)
2115 return false;
2116
2117 // Check that the indices are consecutive, in the case of a multi-byte element
2118 // splatted with a v16i8 mask.
2119 for (unsigned i = 1; i != EltSize; ++i)
2120 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase))
2121 return false;
2122
2123 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
2124 if (N->getMaskElt(i) < 0) continue;
2125 for (unsigned j = 0; j != EltSize; ++j)
2126 if (N->getMaskElt(i+j) != N->getMaskElt(j))
2127 return false;
2128 }
2129 return true;
2130}
2131
2132/// Check that the mask is shuffling N byte elements. Within each N byte
2133/// element of the mask, the indices could be either in increasing or
2134/// decreasing order as long as they are consecutive.
2135/// \param[in] N the shuffle vector SD Node to analyze
2136/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/
2137/// Word/DoubleWord/QuadWord).
2138/// \param[in] StepLen the delta indices number among the N byte element, if
2139/// the mask is in increasing/decreasing order then it is 1/-1.
2140/// \return true iff the mask is shuffling N byte elements.
2141static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width,
2142 int StepLen) {
2143 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) &&(static_cast <bool> ((Width == 2 || Width == 4 || Width
== 8 || Width == 16) && "Unexpected element width.")
? void (0) : __assert_fail ("(Width == 2 || Width == 4 || Width == 8 || Width == 16) && \"Unexpected element width.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2144, __extension__ __PRETTY_FUNCTION__))
2144 "Unexpected element width.")(static_cast <bool> ((Width == 2 || Width == 4 || Width
== 8 || Width == 16) && "Unexpected element width.")
? void (0) : __assert_fail ("(Width == 2 || Width == 4 || Width == 8 || Width == 16) && \"Unexpected element width.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2144, __extension__ __PRETTY_FUNCTION__))
;
2145 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width.")(static_cast <bool> ((StepLen == 1 || StepLen == -1) &&
"Unexpected element width.") ? void (0) : __assert_fail ("(StepLen == 1 || StepLen == -1) && \"Unexpected element width.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2145, __extension__ __PRETTY_FUNCTION__))
;
2146
2147 unsigned NumOfElem = 16 / Width;
2148 unsigned MaskVal[16]; // Width is never greater than 16
2149 for (unsigned i = 0; i < NumOfElem; ++i) {
2150 MaskVal[0] = N->getMaskElt(i * Width);
2151 if ((StepLen == 1) && (MaskVal[0] % Width)) {
2152 return false;
2153 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) {
2154 return false;
2155 }
2156
2157 for (unsigned int j = 1; j < Width; ++j) {
2158 MaskVal[j] = N->getMaskElt(i * Width + j);
2159 if (MaskVal[j] != MaskVal[j-1] + StepLen) {
2160 return false;
2161 }
2162 }
2163 }
2164
2165 return true;
2166}
2167
2168bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2169 unsigned &InsertAtByte, bool &Swap, bool IsLE) {
2170 if (!isNByteElemShuffleMask(N, 4, 1))
2171 return false;
2172
2173 // Now we look at mask elements 0,4,8,12
2174 unsigned M0 = N->getMaskElt(0) / 4;
2175 unsigned M1 = N->getMaskElt(4) / 4;
2176 unsigned M2 = N->getMaskElt(8) / 4;
2177 unsigned M3 = N->getMaskElt(12) / 4;
2178 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 };
2179 unsigned BigEndianShifts[] = { 3, 0, 1, 2 };
2180
2181 // Below, let H and L be arbitrary elements of the shuffle mask
2182 // where H is in the range [4,7] and L is in the range [0,3].
2183 // H, 1, 2, 3 or L, 5, 6, 7
2184 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) ||
2185 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) {
2186 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3];
2187 InsertAtByte = IsLE ? 12 : 0;
2188 Swap = M0 < 4;
2189 return true;
2190 }
2191 // 0, H, 2, 3 or 4, L, 6, 7
2192 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) ||
2193 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) {
2194 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3];
2195 InsertAtByte = IsLE ? 8 : 4;
2196 Swap = M1 < 4;
2197 return true;
2198 }
2199 // 0, 1, H, 3 or 4, 5, L, 7
2200 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) ||
2201 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) {
2202 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3];
2203 InsertAtByte = IsLE ? 4 : 8;
2204 Swap = M2 < 4;
2205 return true;
2206 }
2207 // 0, 1, 2, H or 4, 5, 6, L
2208 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) ||
2209 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) {
2210 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3];
2211 InsertAtByte = IsLE ? 0 : 12;
2212 Swap = M3 < 4;
2213 return true;
2214 }
2215
2216 // If both vector operands for the shuffle are the same vector, the mask will
2217 // contain only elements from the first one and the second one will be undef.
2218 if (N->getOperand(1).isUndef()) {
2219 ShiftElts = 0;
2220 Swap = true;
2221 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1;
2222 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) {
2223 InsertAtByte = IsLE ? 12 : 0;
2224 return true;
2225 }
2226 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) {
2227 InsertAtByte = IsLE ? 8 : 4;
2228 return true;
2229 }
2230 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) {
2231 InsertAtByte = IsLE ? 4 : 8;
2232 return true;
2233 }
2234 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) {
2235 InsertAtByte = IsLE ? 0 : 12;
2236 return true;
2237 }
2238 }
2239
2240 return false;
2241}
2242
2243bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2244 bool &Swap, bool IsLE) {
2245 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8")(static_cast <bool> (N->getValueType(0) == MVT::v16i8
&& "Shuffle vector expects v16i8") ? void (0) : __assert_fail
("N->getValueType(0) == MVT::v16i8 && \"Shuffle vector expects v16i8\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2245, __extension__ __PRETTY_FUNCTION__))
;
2246 // Ensure each byte index of the word is consecutive.
2247 if (!isNByteElemShuffleMask(N, 4, 1))
2248 return false;
2249
2250 // Now we look at mask elements 0,4,8,12, which are the beginning of words.
2251 unsigned M0 = N->getMaskElt(0) / 4;
2252 unsigned M1 = N->getMaskElt(4) / 4;
2253 unsigned M2 = N->getMaskElt(8) / 4;
2254 unsigned M3 = N->getMaskElt(12) / 4;
2255
2256 // If both vector operands for the shuffle are the same vector, the mask will
2257 // contain only elements from the first one and the second one will be undef.
2258 if (N->getOperand(1).isUndef()) {
2259 assert(M0 < 4 && "Indexing into an undef vector?")(static_cast <bool> (M0 < 4 && "Indexing into an undef vector?"
) ? void (0) : __assert_fail ("M0 < 4 && \"Indexing into an undef vector?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2259, __extension__ __PRETTY_FUNCTION__))
;
2260 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4)
2261 return false;
2262
2263 ShiftElts = IsLE ? (4 - M0) % 4 : M0;
2264 Swap = false;
2265 return true;
2266 }
2267
2268 // Ensure each word index of the ShuffleVector Mask is consecutive.
2269 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8)
2270 return false;
2271
2272 if (IsLE) {
2273 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) {
2274 // Input vectors don't need to be swapped if the leading element
2275 // of the result is one of the 3 left elements of the second vector
2276 // (or if there is no shift to be done at all).
2277 Swap = false;
2278 ShiftElts = (8 - M0) % 8;
2279 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) {
2280 // Input vectors need to be swapped if the leading element
2281 // of the result is one of the 3 left elements of the first vector
2282 // (or if we're shifting by 4 - thereby simply swapping the vectors).
2283 Swap = true;
2284 ShiftElts = (4 - M0) % 4;
2285 }
2286
2287 return true;
2288 } else { // BE
2289 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) {
2290 // Input vectors don't need to be swapped if the leading element
2291 // of the result is one of the 4 elements of the first vector.
2292 Swap = false;
2293 ShiftElts = M0;
2294 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) {
2295 // Input vectors need to be swapped if the leading element
2296 // of the result is one of the 4 elements of the right vector.
2297 Swap = true;
2298 ShiftElts = M0 - 4;
2299 }
2300
2301 return true;
2302 }
2303}
2304
2305bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) {
2306 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8")(static_cast <bool> (N->getValueType(0) == MVT::v16i8
&& "Shuffle vector expects v16i8") ? void (0) : __assert_fail
("N->getValueType(0) == MVT::v16i8 && \"Shuffle vector expects v16i8\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2306, __extension__ __PRETTY_FUNCTION__))
;
2307
2308 if (!isNByteElemShuffleMask(N, Width, -1))
2309 return false;
2310
2311 for (int i = 0; i < 16; i += Width)
2312 if (N->getMaskElt(i) != i + Width - 1)
2313 return false;
2314
2315 return true;
2316}
2317
2318bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) {
2319 return isXXBRShuffleMaskHelper(N, 2);
2320}
2321
2322bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) {
2323 return isXXBRShuffleMaskHelper(N, 4);
2324}
2325
2326bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) {
2327 return isXXBRShuffleMaskHelper(N, 8);
2328}
2329
2330bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) {
2331 return isXXBRShuffleMaskHelper(N, 16);
2332}
2333
2334/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap
2335/// if the inputs to the instruction should be swapped and set \p DM to the
2336/// value for the immediate.
2337/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI
2338/// AND element 0 of the result comes from the first input (LE) or second input
2339/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered.
2340/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle
2341/// mask.
2342bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
2343 bool &Swap, bool IsLE) {
2344 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8")(static_cast <bool> (N->getValueType(0) == MVT::v16i8
&& "Shuffle vector expects v16i8") ? void (0) : __assert_fail
("N->getValueType(0) == MVT::v16i8 && \"Shuffle vector expects v16i8\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2344, __extension__ __PRETTY_FUNCTION__))
;
2345
2346 // Ensure each byte index of the double word is consecutive.
2347 if (!isNByteElemShuffleMask(N, 8, 1))
2348 return false;
2349
2350 unsigned M0 = N->getMaskElt(0) / 8;
2351 unsigned M1 = N->getMaskElt(8) / 8;
2352 assert(((M0 | M1) < 4) && "A mask element out of bounds?")(static_cast <bool> (((M0 | M1) < 4) && "A mask element out of bounds?"
) ? void (0) : __assert_fail ("((M0 | M1) < 4) && \"A mask element out of bounds?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2352, __extension__ __PRETTY_FUNCTION__))
;
2353
2354 // If both vector operands for the shuffle are the same vector, the mask will
2355 // contain only elements from the first one and the second one will be undef.
2356 if (N->getOperand(1).isUndef()) {
2357 if ((M0 | M1) < 2) {
2358 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1);
2359 Swap = false;
2360 return true;
2361 } else
2362 return false;
2363 }
2364
2365 if (IsLE) {
2366 if (M0 > 1 && M1 < 2) {
2367 Swap = false;
2368 } else if (M0 < 2 && M1 > 1) {
2369 M0 = (M0 + 2) % 4;
2370 M1 = (M1 + 2) % 4;
2371 Swap = true;
2372 } else
2373 return false;
2374
2375 // Note: if control flow comes here that means Swap is already set above
2376 DM = (((~M1) & 1) << 1) + ((~M0) & 1);
2377 return true;
2378 } else { // BE
2379 if (M0 < 2 && M1 > 1) {
2380 Swap = false;
2381 } else if (M0 > 1 && M1 < 2) {
2382 M0 = (M0 + 2) % 4;
2383 M1 = (M1 + 2) % 4;
2384 Swap = true;
2385 } else
2386 return false;
2387
2388 // Note: if control flow comes here that means Swap is already set above
2389 DM = (M0 << 1) + (M1 & 1);
2390 return true;
2391 }
2392}
2393
2394
2395/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
2396/// appropriate for PPC mnemonics (which have a big endian bias - namely
2397/// elements are counted from the left of the vector register).
2398unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
2399 SelectionDAG &DAG) {
2400 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
2401 assert(isSplatShuffleMask(SVOp, EltSize))(static_cast <bool> (isSplatShuffleMask(SVOp, EltSize))
? void (0) : __assert_fail ("isSplatShuffleMask(SVOp, EltSize)"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2401, __extension__ __PRETTY_FUNCTION__))
;
2402 if (DAG.getDataLayout().isLittleEndian())
2403 return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize);
2404 else
2405 return SVOp->getMaskElt(0) / EltSize;
2406}
2407
2408/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
2409/// by using a vspltis[bhw] instruction of the specified element size, return
2410/// the constant being splatted. The ByteSize field indicates the number of
2411/// bytes of each element [124] -> [bhw].
2412SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
2413 SDValue OpVal(nullptr, 0);
2414
2415 // If ByteSize of the splat is bigger than the element size of the
2416 // build_vector, then we have a case where we are checking for a splat where
2417 // multiple elements of the buildvector are folded together into a single
2418 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
2419 unsigned EltSize = 16/N->getNumOperands();
2420 if (EltSize < ByteSize) {
2421 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval.
2422 SDValue UniquedVals[4];
2423 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?")(static_cast <bool> (Multiple > 1 && Multiple
<= 4 && "How can this happen?") ? void (0) : __assert_fail
("Multiple > 1 && Multiple <= 4 && \"How can this happen?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2423, __extension__ __PRETTY_FUNCTION__))
;
2424
2425 // See if all of the elements in the buildvector agree across.
2426 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2427 if (N->getOperand(i).isUndef()) continue;
2428 // If the element isn't a constant, bail fully out.
2429 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue();
2430
2431 if (!UniquedVals[i&(Multiple-1)].getNode())
2432 UniquedVals[i&(Multiple-1)] = N->getOperand(i);
2433 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i))
2434 return SDValue(); // no match.
2435 }
2436
2437 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
2438 // either constant or undef values that are identical for each chunk. See
2439 // if these chunks can form into a larger vspltis*.
2440
2441 // Check to see if all of the leading entries are either 0 or -1. If
2442 // neither, then this won't fit into the immediate field.
2443 bool LeadingZero = true;
2444 bool LeadingOnes = true;
2445 for (unsigned i = 0; i != Multiple-1; ++i) {
2446 if (!UniquedVals[i].getNode()) continue; // Must have been undefs.
2447
2448 LeadingZero &= isNullConstant(UniquedVals[i]);
2449 LeadingOnes &= isAllOnesConstant(UniquedVals[i]);
2450 }
2451 // Finally, check the least significant entry.
2452 if (LeadingZero) {
2453 if (!UniquedVals[Multiple-1].getNode())
2454 return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef
2455 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue();
2456 if (Val < 16) // 0,0,0,4 -> vspltisw(4)
2457 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
2458 }
2459 if (LeadingOnes) {
2460 if (!UniquedVals[Multiple-1].getNode())
2461 return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef
2462 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue();
2463 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2)
2464 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
2465 }
2466
2467 return SDValue();
2468 }
2469
2470 // Check to see if this buildvec has a single non-undef value in its elements.
2471 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2472 if (N->getOperand(i).isUndef()) continue;
2473 if (!OpVal.getNode())
2474 OpVal = N->getOperand(i);
2475 else if (OpVal != N->getOperand(i))
2476 return SDValue();
2477 }
2478
2479 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def.
2480
2481 unsigned ValSizeInBytes = EltSize;
2482 uint64_t Value = 0;
2483 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) {
2484 Value = CN->getZExtValue();
2485 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) {
2486 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!")(static_cast <bool> (CN->getValueType(0) == MVT::f32
&& "Only one legal FP vector type!") ? void (0) : __assert_fail
("CN->getValueType(0) == MVT::f32 && \"Only one legal FP vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2486, __extension__ __PRETTY_FUNCTION__))
;
2487 Value = FloatToBits(CN->getValueAPF().convertToFloat());
2488 }
2489
2490 // If the splat value is larger than the element value, then we can never do
2491 // this splat. The only case that we could fit the replicated bits into our
2492 // immediate field for would be zero, and we prefer to use vxor for it.
2493 if (ValSizeInBytes < ByteSize) return SDValue();
2494
2495 // If the element value is larger than the splat value, check if it consists
2496 // of a repeated bit pattern of size ByteSize.
2497 if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8))
2498 return SDValue();
2499
2500 // Properly sign extend the value.
2501 int MaskVal = SignExtend32(Value, ByteSize * 8);
2502
2503 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
2504 if (MaskVal == 0) return SDValue();
2505
2506 // Finally, if this value fits in a 5 bit sext field, return it
2507 if (SignExtend32<5>(MaskVal) == MaskVal)
2508 return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32);
2509 return SDValue();
2510}
2511
2512//===----------------------------------------------------------------------===//
2513// Addressing Mode Selection
2514//===----------------------------------------------------------------------===//
2515
2516/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
2517/// or 64-bit immediate, and if the value can be accurately represented as a
2518/// sign extension from a 16-bit value. If so, this returns true and the
2519/// immediate.
2520bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) {
2521 if (!isa<ConstantSDNode>(N))
2522 return false;
2523
2524 Imm = (int16_t)cast<ConstantSDNode>(N)->getZExtValue();
2525 if (N->getValueType(0) == MVT::i32)
2526 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue();
2527 else
2528 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue();
2529}
2530bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
2531 return isIntS16Immediate(Op.getNode(), Imm);
2532}
2533
2534/// Used when computing address flags for selecting loads and stores.
2535/// If we have an OR, check if the LHS and RHS are provably disjoint.
2536/// An OR of two provably disjoint values is equivalent to an ADD.
2537/// Most PPC load/store instructions compute the effective address as a sum,
2538/// so doing this conversion is useful.
2539static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) {
2540 if (N.getOpcode() != ISD::OR)
2541 return false;
2542 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
2543 if (!LHSKnown.Zero.getBoolValue())
2544 return false;
2545 KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1));
2546 return (~(LHSKnown.Zero | RHSKnown.Zero) == 0);
2547}
2548
2549/// SelectAddressEVXRegReg - Given the specified address, check to see if it can
2550/// be represented as an indexed [r+r] operation.
2551bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2552 SDValue &Index,
2553 SelectionDAG &DAG) const {
2554 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
2555 UI != E; ++UI) {
2556 if (MemSDNode *Memop = dyn_cast<MemSDNode>(*UI)) {
2557 if (Memop->getMemoryVT() == MVT::f64) {
2558 Base = N.getOperand(0);
2559 Index = N.getOperand(1);
2560 return true;
2561 }
2562 }
2563 }
2564 return false;
2565}
2566
2567/// isIntS34Immediate - This method tests if value of node given can be
2568/// accurately represented as a sign extension from a 34-bit value. If so,
2569/// this returns true and the immediate.
2570bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
2571 if (!isa<ConstantSDNode>(N))
2572 return false;
2573
2574 Imm = (int64_t)cast<ConstantSDNode>(N)->getZExtValue();
2575 return isInt<34>(Imm);
2576}
2577bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
2578 return isIntS34Immediate(Op.getNode(), Imm);
2579}
2580
2581/// SelectAddressRegReg - Given the specified addressed, check to see if it
2582/// can be represented as an indexed [r+r] operation. Returns false if it
2583/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is
2584/// non-zero and N can be represented by a base register plus a signed 16-bit
2585/// displacement, make a more precise judgement by checking (displacement % \p
2586/// EncodingAlignment).
2587bool PPCTargetLowering::SelectAddressRegReg(
2588 SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
2589 MaybeAlign EncodingAlignment) const {
2590 // If we have a PC Relative target flag don't select as [reg+reg]. It will be
2591 // a [pc+imm].
2592 if (SelectAddressPCRel(N, Base))
2593 return false;
2594
2595 int16_t Imm = 0;
2596 if (N.getOpcode() == ISD::ADD) {
2597 // Is there any SPE load/store (f64), which can't handle 16bit offset?
2598 // SPE load/store can only handle 8-bit offsets.
2599 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
2600 return true;
2601 if (isIntS16Immediate(N.getOperand(1), Imm) &&
2602 (!EncodingAlignment || isAligned(*EncodingAlignment, Imm)))
2603 return false; // r+i
2604 if (N.getOperand(1).getOpcode() == PPCISD::Lo)
2605 return false; // r+i
2606
2607 Base = N.getOperand(0);
2608 Index = N.getOperand(1);
2609 return true;
2610 } else if (N.getOpcode() == ISD::OR) {
2611 if (isIntS16Immediate(N.getOperand(1), Imm) &&
2612 (!EncodingAlignment || isAligned(*EncodingAlignment, Imm)))
2613 return false; // r+i can fold it if we can.
2614
2615 // If this is an or of disjoint bitfields, we can codegen this as an add
2616 // (for better address arithmetic) if the LHS and RHS of the OR are provably
2617 // disjoint.
2618 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
2619
2620 if (LHSKnown.Zero.getBoolValue()) {
2621 KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1));
2622 // If all of the bits are known zero on the LHS or RHS, the add won't
2623 // carry.
2624 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
2625 Base = N.getOperand(0);
2626 Index = N.getOperand(1);
2627 return true;
2628 }
2629 }
2630 }
2631
2632 return false;
2633}
2634
2635// If we happen to be doing an i64 load or store into a stack slot that has
2636// less than a 4-byte alignment, then the frame-index elimination may need to
2637// use an indexed load or store instruction (because the offset may not be a
2638// multiple of 4). The extra register needed to hold the offset comes from the
2639// register scavenger, and it is possible that the scavenger will need to use
2640// an emergency spill slot. As a result, we need to make sure that a spill slot
2641// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
2642// stack slot.
2643static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
2644 // FIXME: This does not handle the LWA case.
2645 if (VT != MVT::i64)
2646 return;
2647
2648 // NOTE: We'll exclude negative FIs here, which come from argument
2649 // lowering, because there are no known test cases triggering this problem
2650 // using packed structures (or similar). We can remove this exclusion if
2651 // we find such a test case. The reason why this is so test-case driven is
2652 // because this entire 'fixup' is only to prevent crashes (from the
2653 // register scavenger) on not-really-valid inputs. For example, if we have:
2654 // %a = alloca i1
2655 // %b = bitcast i1* %a to i64*
2656 // store i64* a, i64 b
2657 // then the store should really be marked as 'align 1', but is not. If it
2658 // were marked as 'align 1' then the indexed form would have been
2659 // instruction-selected initially, and the problem this 'fixup' is preventing
2660 // won't happen regardless.
2661 if (FrameIdx < 0)
2662 return;
2663
2664 MachineFunction &MF = DAG.getMachineFunction();
2665 MachineFrameInfo &MFI = MF.getFrameInfo();
2666
2667 if (MFI.getObjectAlign(FrameIdx) >= Align(4))
2668 return;
2669
2670 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2671 FuncInfo->setHasNonRISpills();
2672}
2673
2674/// Returns true if the address N can be represented by a base register plus
2675/// a signed 16-bit displacement [r+imm], and if it is not better
2676/// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept
2677/// displacements that are multiples of that value.
2678bool PPCTargetLowering::SelectAddressRegImm(
2679 SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
2680 MaybeAlign EncodingAlignment) const {
2681 // FIXME dl should come from parent load or store, not from address
2682 SDLoc dl(N);
2683
2684 // If we have a PC Relative target flag don't select as [reg+imm]. It will be
2685 // a [pc+imm].
2686 if (SelectAddressPCRel(N, Base))
2687 return false;
2688
2689 // If this can be more profitably realized as r+r, fail.
2690 if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment))
2691 return false;
2692
2693 if (N.getOpcode() == ISD::ADD) {
2694 int16_t imm = 0;
2695 if (isIntS16Immediate(N.getOperand(1), imm) &&
2696 (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) {
2697 Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
2698 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
2699 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
2700 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
2701 } else {
2702 Base = N.getOperand(0);
2703 }
2704 return true; // [r+i]
2705 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) {
2706 // Match LOAD (ADD (X, Lo(G))).
2707 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue()(static_cast <bool> (!cast<ConstantSDNode>(N.getOperand
(1).getOperand(1))->getZExtValue() && "Cannot handle constant offsets yet!"
) ? void (0) : __assert_fail ("!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() && \"Cannot handle constant offsets yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2708, __extension__ __PRETTY_FUNCTION__))
2708 && "Cannot handle constant offsets yet!")(static_cast <bool> (!cast<ConstantSDNode>(N.getOperand
(1).getOperand(1))->getZExtValue() && "Cannot handle constant offsets yet!"
) ? void (0) : __assert_fail ("!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() && \"Cannot handle constant offsets yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2708, __extension__ __PRETTY_FUNCTION__))
;
2709 Disp = N.getOperand(1).getOperand(0); // The global address.
2710 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2713, __extension__ __PRETTY_FUNCTION__))
2711 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2713, __extension__ __PRETTY_FUNCTION__))
2712 Disp.getOpcode() == ISD::TargetConstantPool ||(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2713, __extension__ __PRETTY_FUNCTION__))
2713 Disp.getOpcode() == ISD::TargetJumpTable)(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 2713, __extension__ __PRETTY_FUNCTION__))
;
2714 Base = N.getOperand(0);
2715 return true; // [&g+r]
2716 }
2717 } else if (N.getOpcode() == ISD::OR) {
2718 int16_t imm = 0;
2719 if (isIntS16Immediate(N.getOperand(1), imm) &&
2720 (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) {
2721 // If this is an or of disjoint bitfields, we can codegen this as an add
2722 // (for better address arithmetic) if the LHS and RHS of the OR are
2723 // provably disjoint.
2724 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
2725
2726 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
2727 // If all of the bits are known zero on the LHS or RHS, the add won't
2728 // carry.
2729 if (FrameIndexSDNode *FI =
2730 dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
2731 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
2732 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
2733 } else {
2734 Base = N.getOperand(0);
2735 }
2736 Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
2737 return true;
2738 }
2739 }
2740 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) {
2741 // Loading from a constant address.
2742
2743 // If this address fits entirely in a 16-bit sext immediate field, codegen
2744 // this as "d, 0"
2745 int16_t Imm;
2746 if (isIntS16Immediate(CN, Imm) &&
2747 (!EncodingAlignment || isAligned(*EncodingAlignment, Imm))) {
2748 Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0));
2749 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2750 CN->getValueType(0));
2751 return true;
2752 }
2753
2754 // Handle 32-bit sext immediates with LIS + addr mode.
2755 if ((CN->getValueType(0) == MVT::i32 ||
2756 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
2757 (!EncodingAlignment ||
2758 isAligned(*EncodingAlignment, CN->getZExtValue()))) {
2759 int Addr = (int)CN->getZExtValue();
2760
2761 // Otherwise, break this down into an LIS + disp.
2762 Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32);
2763
2764 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl,
2765 MVT::i32);
2766 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
2767 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0);
2768 return true;
2769 }
2770 }
2771
2772 Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout()));
2773 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) {
2774 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
2775 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
2776 } else
2777 Base = N;
2778 return true; // [r+0]
2779}
2780
2781/// Similar to the 16-bit case but for instructions that take a 34-bit
2782/// displacement field (prefixed loads/stores).
2783bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
2784 SDValue &Base,
2785 SelectionDAG &DAG) const {
2786 // Only on 64-bit targets.
2787 if (N.getValueType() != MVT::i64)
2788 return false;
2789
2790 SDLoc dl(N);
2791 int64_t Imm = 0;
2792
2793 if (N.getOpcode() == ISD::ADD) {
2794 if (!isIntS34Immediate(N.getOperand(1), Imm))
2795 return false;
2796 Disp = DAG.getTargetConstant(Imm, dl, N.getValueType());
2797 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0)))
2798 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
2799 else
2800 Base = N.getOperand(0);
2801 return true;
2802 }
2803
2804 if (N.getOpcode() == ISD::OR) {
2805 if (!isIntS34Immediate(N.getOperand(1), Imm))
2806 return false;
2807 // If this is an or of disjoint bitfields, we can codegen this as an add
2808 // (for better address arithmetic) if the LHS and RHS of the OR are
2809 // provably disjoint.
2810 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
2811 if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL)
2812 return false;
2813 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0)))
2814 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
2815 else
2816 Base = N.getOperand(0);
2817 Disp = DAG.getTargetConstant(Imm, dl, N.getValueType());
2818 return true;
2819 }
2820
2821 if (isIntS34Immediate(N, Imm)) { // If the address is a 34-bit const.
2822 Disp = DAG.getTargetConstant(Imm, dl, N.getValueType());
2823 Base = DAG.getRegister(PPC::ZERO8, N.getValueType());
2824 return true;
2825 }
2826
2827 return false;
2828}
2829
2830/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
2831/// represented as an indexed [r+r] operation.
2832bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2833 SDValue &Index,
2834 SelectionDAG &DAG) const {
2835 // Check to see if we can easily represent this as an [r+r] address. This
2836 // will fail if it thinks that the address is more profitably represented as
2837 // reg+imm, e.g. where imm = 0.
2838 if (SelectAddressRegReg(N, Base, Index, DAG))
2839 return true;
2840
2841 // If the address is the result of an add, we will utilize the fact that the
2842 // address calculation includes an implicit add. However, we can reduce
2843 // register pressure if we do not materialize a constant just for use as the
2844 // index register. We only get rid of the add if it is not an add of a
2845 // value and a 16-bit signed constant and both have a single use.
2846 int16_t imm = 0;
2847 if (N.getOpcode() == ISD::ADD &&
2848 (!isIntS16Immediate(N.getOperand(1), imm) ||
2849 !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) {
2850 Base = N.getOperand(0);
2851 Index = N.getOperand(1);
2852 return true;
2853 }
2854
2855 // Otherwise, do it the hard way, using R0 as the base register.
2856 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2857 N.getValueType());
2858 Index = N;
2859 return true;
2860}
2861
2862template <typename Ty> static bool isValidPCRelNode(SDValue N) {
2863 Ty *PCRelCand = dyn_cast<Ty>(N);
2864 return PCRelCand && (PCRelCand->getTargetFlags() & PPCII::MO_PCREL_FLAG);
2865}
2866
2867/// Returns true if this address is a PC Relative address.
2868/// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG
2869/// or if the node opcode is PPCISD::MAT_PCREL_ADDR.
2870bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const {
2871 // This is a materialize PC Relative node. Always select this as PC Relative.
2872 Base = N;
2873 if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR)
2874 return true;
2875 if (isValidPCRelNode<ConstantPoolSDNode>(N) ||
2876 isValidPCRelNode<GlobalAddressSDNode>(N) ||
2877 isValidPCRelNode<JumpTableSDNode>(N) ||
2878 isValidPCRelNode<BlockAddressSDNode>(N))
2879 return true;
2880 return false;
2881}
2882
2883/// Returns true if we should use a direct load into vector instruction
2884/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence.
2885static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) {
2886
2887 // If there are any other uses other than scalar to vector, then we should
2888 // keep it as a scalar load -> direct move pattern to prevent multiple
2889 // loads.
2890 LoadSDNode *LD = dyn_cast<LoadSDNode>(N);
2891 if (!LD)
2892 return false;
2893
2894 EVT MemVT = LD->getMemoryVT();
2895 if (!MemVT.isSimple())
2896 return false;
2897 switch(MemVT.getSimpleVT().SimpleTy) {
2898 case MVT::i64:
2899 break;
2900 case MVT::i32:
2901 if (!ST.hasP8Vector())
2902 return false;
2903 break;
2904 case MVT::i16:
2905 case MVT::i8:
2906 if (!ST.hasP9Vector())
2907 return false;
2908 break;
2909 default:
2910 return false;
2911 }
2912
2913 SDValue LoadedVal(N, 0);
2914 if (!LoadedVal.hasOneUse())
2915 return false;
2916
2917 for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end();
2918 UI != UE; ++UI)
2919 if (UI.getUse().get().getResNo() == 0 &&
2920 UI->getOpcode() != ISD::SCALAR_TO_VECTOR &&
2921 UI->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED)
2922 return false;
2923
2924 return true;
2925}
2926
2927/// getPreIndexedAddressParts - returns true by value, base pointer and
2928/// offset pointer and addressing mode by reference if the node's address
2929/// can be legally represented as pre-indexed load / store address.
2930bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
2931 SDValue &Offset,
2932 ISD::MemIndexedMode &AM,
2933 SelectionDAG &DAG) const {
2934 if (DisablePPCPreinc) return false;
2935
2936 bool isLoad = true;
2937 SDValue Ptr;
2938 EVT VT;
2939 unsigned Alignment;
2940 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
2941 Ptr = LD->getBasePtr();
2942 VT = LD->getMemoryVT();
2943 Alignment = LD->getAlignment();
2944 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
2945 Ptr = ST->getBasePtr();
2946 VT = ST->getMemoryVT();
2947 Alignment = ST->getAlignment();
2948 isLoad = false;
2949 } else
2950 return false;
2951
2952 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
2953 // instructions because we can fold these into a more efficient instruction
2954 // instead, (such as LXSD).
2955 if (isLoad && usePartialVectorLoads(N, Subtarget)) {
2956 return false;
2957 }
2958
2959 // PowerPC doesn't have preinc load/store instructions for vectors
2960 if (VT.isVector())
2961 return false;
2962
2963 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) {
2964 // Common code will reject creating a pre-inc form if the base pointer
2965 // is a frame index, or if N is a store and the base pointer is either
2966 // the same as or a predecessor of the value being stored. Check for
2967 // those situations here, and try with swapped Base/Offset instead.
2968 bool Swap = false;
2969
2970 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base))
2971 Swap = true;
2972 else if (!isLoad) {
2973 SDValue Val = cast<StoreSDNode>(N)->getValue();
2974 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode()))
2975 Swap = true;
2976 }
2977
2978 if (Swap)
2979 std::swap(Base, Offset);
2980
2981 AM = ISD::PRE_INC;
2982 return true;
2983 }
2984
2985 // LDU/STU can only handle immediates that are a multiple of 4.
2986 if (VT != MVT::i64) {
2987 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, None))
2988 return false;
2989 } else {
2990 // LDU/STU need an address with at least 4-byte alignment.
2991 if (Alignment < 4)
2992 return false;
2993
2994 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, Align(4)))
2995 return false;
2996 }
2997
2998 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
2999 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of
3000 // sext i32 to i64 when addr mode is r+i.
3001 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
3002 LD->getExtensionType() == ISD::SEXTLOAD &&
3003 isa<ConstantSDNode>(Offset))
3004 return false;
3005 }
3006
3007 AM = ISD::PRE_INC;
3008 return true;
3009}
3010
3011//===----------------------------------------------------------------------===//
3012// LowerOperation implementation
3013//===----------------------------------------------------------------------===//
3014
3015/// Return true if we should reference labels using a PICBase, set the HiOpFlags
3016/// and LoOpFlags to the target MO flags.
3017static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget,
3018 unsigned &HiOpFlags, unsigned &LoOpFlags,
3019 const GlobalValue *GV = nullptr) {
3020 HiOpFlags = PPCII::MO_HA;
3021 LoOpFlags = PPCII::MO_LO;
3022
3023 // Don't use the pic base if not in PIC relocation model.
3024 if (IsPIC) {
3025 HiOpFlags |= PPCII::MO_PIC_FLAG;
3026 LoOpFlags |= PPCII::MO_PIC_FLAG;
3027 }
3028}
3029
3030static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
3031 SelectionDAG &DAG) {
3032 SDLoc DL(HiPart);
3033 EVT PtrVT = HiPart.getValueType();
3034 SDValue Zero = DAG.getConstant(0, DL, PtrVT);
3035
3036 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero);
3037 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero);
3038
3039 // With PIC, the first instruction is actually "GR+hi(&G)".
3040 if (isPIC)
3041 Hi = DAG.getNode(ISD::ADD, DL, PtrVT,
3042 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi);
3043
3044 // Generate non-pic code that has direct accesses to the constant pool.
3045 // The address of the global is just (hi(&g)+lo(&g)).
3046 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo);
3047}
3048
3049static void setUsesTOCBasePtr(MachineFunction &MF) {
3050 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3051 FuncInfo->setUsesTOCBasePtr();
3052}
3053
3054static void setUsesTOCBasePtr(SelectionDAG &DAG) {
3055 setUsesTOCBasePtr(DAG.getMachineFunction());
3056}
3057
3058SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
3059 SDValue GA) const {
3060 const bool Is64Bit = Subtarget.isPPC64();
3061 EVT VT = Is64Bit ? MVT::i64 : MVT::i32;
3062 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT)
3063 : Subtarget.isAIXABI()
3064 ? DAG.getRegister(PPC::R2, VT)
3065 : DAG.getNode(PPCISD::GlobalBaseReg, dl, VT);
3066 SDValue Ops[] = { GA, Reg };
3067 return DAG.getMemIntrinsicNode(
3068 PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT,
3069 MachinePointerInfo::getGOT(DAG.getMachineFunction()), None,
3070 MachineMemOperand::MOLoad);
3071}
3072
3073SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
3074 SelectionDAG &DAG) const {
3075 EVT PtrVT = Op.getValueType();
3076 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
3077 const Constant *C = CP->getConstVal();
3078
3079 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3080 // The actual address of the GlobalValue is stored in the TOC.
3081 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3082 if (Subtarget.isUsingPCRelativeCalls()) {
3083 SDLoc DL(CP);
3084 EVT Ty = getPointerTy(DAG.getDataLayout());
3085 SDValue ConstPool = DAG.getTargetConstantPool(
3086 C, Ty, CP->getAlign(), CP->getOffset(), PPCII::MO_PCREL_FLAG);
3087 return DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, ConstPool);
3088 }
3089 setUsesTOCBasePtr(DAG);
3090 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0);
3091 return getTOCEntry(DAG, SDLoc(CP), GA);
3092 }
3093
3094 unsigned MOHiFlag, MOLoFlag;
3095 bool IsPIC = isPositionIndependent();
3096 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag);
3097
3098 if (IsPIC && Subtarget.isSVR4ABI()) {
3099 SDValue GA =
3100 DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), PPCII::MO_PIC_FLAG);
3101 return getTOCEntry(DAG, SDLoc(CP), GA);
3102 }
3103
3104 SDValue CPIHi =
3105 DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0, MOHiFlag);
3106 SDValue CPILo =
3107 DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0, MOLoFlag);
3108 return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG);
3109}
3110
3111// For 64-bit PowerPC, prefer the more compact relative encodings.
3112// This trades 32 bits per jump table entry for one or two instructions
3113// on the jump site.
3114unsigned PPCTargetLowering::getJumpTableEncoding() const {
3115 if (isJumpTableRelative())
3116 return MachineJumpTableInfo::EK_LabelDifference32;
3117
3118 return TargetLowering::getJumpTableEncoding();
3119}
3120
3121bool PPCTargetLowering::isJumpTableRelative() const {
3122 if (UseAbsoluteJumpTables)
3123 return false;
3124 if (Subtarget.isPPC64() || Subtarget.isAIXABI())
3125 return true;
3126 return TargetLowering::isJumpTableRelative();
3127}
3128
3129SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3130 SelectionDAG &DAG) const {
3131 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3132 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3133
3134 switch (getTargetMachine().getCodeModel()) {
3135 case CodeModel::Small:
3136 case CodeModel::Medium:
3137 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3138 default:
3139 return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(),
3140 getPointerTy(DAG.getDataLayout()));
3141 }
3142}
3143
3144const MCExpr *
3145PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
3146 unsigned JTI,
3147 MCContext &Ctx) const {
3148 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3149 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3150
3151 switch (getTargetMachine().getCodeModel()) {
3152 case CodeModel::Small:
3153 case CodeModel::Medium:
3154 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3155 default:
3156 return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
3157 }
3158}
3159
3160SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
3161 EVT PtrVT = Op.getValueType();
3162 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
3163
3164 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3165 if (Subtarget.isUsingPCRelativeCalls()) {
3166 SDLoc DL(JT);
3167 EVT Ty = getPointerTy(DAG.getDataLayout());
3168 SDValue GA =
3169 DAG.getTargetJumpTable(JT->getIndex(), Ty, PPCII::MO_PCREL_FLAG);
3170 SDValue MatAddr = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
3171 return MatAddr;
3172 }
3173
3174 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3175 // The actual address of the GlobalValue is stored in the TOC.
3176 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3177 setUsesTOCBasePtr(DAG);
3178 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
3179 return getTOCEntry(DAG, SDLoc(JT), GA);
3180 }
3181
3182 unsigned MOHiFlag, MOLoFlag;
3183 bool IsPIC = isPositionIndependent();
3184 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag);
3185
3186 if (IsPIC && Subtarget.isSVR4ABI()) {
3187 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT,
3188 PPCII::MO_PIC_FLAG);
3189 return getTOCEntry(DAG, SDLoc(GA), GA);
3190 }
3191
3192 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag);
3193 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag);
3194 return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG);
3195}
3196
3197SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
3198 SelectionDAG &DAG) const {
3199 EVT PtrVT = Op.getValueType();
3200 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op);
3201 const BlockAddress *BA = BASDN->getBlockAddress();
3202
3203 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3204 if (Subtarget.isUsingPCRelativeCalls()) {
3205 SDLoc DL(BASDN);
3206 EVT Ty = getPointerTy(DAG.getDataLayout());
3207 SDValue GA = DAG.getTargetBlockAddress(BA, Ty, BASDN->getOffset(),
3208 PPCII::MO_PCREL_FLAG);
3209 SDValue MatAddr = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
3210 return MatAddr;
3211 }
3212
3213 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3214 // The actual BlockAddress is stored in the TOC.
3215 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3216 setUsesTOCBasePtr(DAG);
3217 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset());
3218 return getTOCEntry(DAG, SDLoc(BASDN), GA);
3219 }
3220
3221 // 32-bit position-independent ELF stores the BlockAddress in the .got.
3222 if (Subtarget.is32BitELFABI() && isPositionIndependent())
3223 return getTOCEntry(
3224 DAG, SDLoc(BASDN),
3225 DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()));
3226
3227 unsigned MOHiFlag, MOLoFlag;
3228 bool IsPIC = isPositionIndependent();
3229 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag);
3230 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag);
3231 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag);
3232 return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG);
3233}
3234
3235SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
3236 SelectionDAG &DAG) const {
3237 if (Subtarget.isAIXABI())
3238 return LowerGlobalTLSAddressAIX(Op, DAG);
3239
3240 return LowerGlobalTLSAddressLinux(Op, DAG);
3241}
3242
3243SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
3244 SelectionDAG &DAG) const {
3245 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
3246
3247 if (DAG.getTarget().useEmulatedTLS())
3248 report_fatal_error("Emulated TLS is not yet supported on AIX");
3249
3250 SDLoc dl(GA);
3251 const GlobalValue *GV = GA->getGlobal();
3252 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3253
3254 // The general-dynamic model is the only access model supported for now, so
3255 // all the GlobalTLSAddress nodes are lowered with this model.
3256 // We need to generate two TOC entries, one for the variable offset, one for
3257 // the region handle. The global address for the TOC entry of the region
3258 // handle is created with the MO_TLSGDM_FLAG flag and the global address
3259 // for the TOC entry of the variable offset is created with MO_TLSGD_FLAG.
3260 SDValue VariableOffsetTGA =
3261 DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGD_FLAG);
3262 SDValue RegionHandleTGA =
3263 DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGDM_FLAG);
3264 SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA);
3265 SDValue RegionHandle = getTOCEntry(DAG, dl, RegionHandleTGA);
3266 return DAG.getNode(PPCISD::TLSGD_AIX, dl, PtrVT, VariableOffset,
3267 RegionHandle);
3268}
3269
3270SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op,
3271 SelectionDAG &DAG) const {
3272 // FIXME: TLS addresses currently use medium model code sequences,
3273 // which is the most useful form. Eventually support for small and
3274 // large models could be added if users need it, at the cost of
3275 // additional complexity.
3276 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
3277 if (DAG.getTarget().useEmulatedTLS())
3278 return LowerToTLSEmulatedModel(GA, DAG);
3279
3280 SDLoc dl(GA);
3281 const GlobalValue *GV = GA->getGlobal();
3282 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3283 bool is64bit = Subtarget.isPPC64();
3284 const Module *M = DAG.getMachineFunction().getFunction().getParent();
3285 PICLevel::Level picLevel = M->getPICLevel();
3286
3287 const TargetMachine &TM = getTargetMachine();
3288 TLSModel::Model Model = TM.getTLSModel(GV);
3289
3290 if (Model == TLSModel::LocalExec) {
3291 if (Subtarget.isUsingPCRelativeCalls()) {
3292 SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64);
3293 SDValue TGA = DAG.getTargetGlobalAddress(
3294 GV, dl, PtrVT, 0, (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG));
3295 SDValue MatAddr =
3296 DAG.getNode(PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, dl, PtrVT, TGA);
3297 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, MatAddr);
3298 }
3299
3300 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3301 PPCII::MO_TPREL_HA);
3302 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3303 PPCII::MO_TPREL_LO);
3304 SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64)
3305 : DAG.getRegister(PPC::R2, MVT::i32);
3306
3307 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg);
3308 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi);
3309 }
3310
3311 if (Model == TLSModel::InitialExec) {
3312 bool IsPCRel = Subtarget.isUsingPCRelativeCalls();
3313 SDValue TGA = DAG.getTargetGlobalAddress(
3314 GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0);
3315 SDValue TGATLS = DAG.getTargetGlobalAddress(
3316 GV, dl, PtrVT, 0,
3317 IsPCRel ? (PPCII::MO_TLS | PPCII::MO_PCREL_FLAG) : PPCII::MO_TLS);
3318 SDValue TPOffset;
3319 if (IsPCRel) {
3320 SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, dl, PtrVT, TGA);
3321 TPOffset = DAG.getLoad(MVT::i64, dl, DAG.getEntryNode(), MatPCRel,
3322 MachinePointerInfo());
3323 } else {
3324 SDValue GOTPtr;
3325 if (is64bit) {
3326 setUsesTOCBasePtr(DAG);
3327 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
3328 GOTPtr =
3329 DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, PtrVT, GOTReg, TGA);
3330 } else {
3331 if (!TM.isPositionIndependent())
3332 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT);
3333 else if (picLevel == PICLevel::SmallPIC)
3334 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
3335 else
3336 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
3337 }
3338 TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, PtrVT, TGA, GOTPtr);
3339 }
3340 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS);
3341 }
3342
3343 if (Model == TLSModel::GeneralDynamic) {
3344 if (Subtarget.isUsingPCRelativeCalls()) {
3345 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3346 PPCII::MO_GOT_TLSGD_PCREL_FLAG);
3347 return DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA);
3348 }
3349
3350 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
3351 SDValue GOTPtr;
3352 if (is64bit) {
3353 setUsesTOCBasePtr(DAG);
3354 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
3355 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT,
3356 GOTReg, TGA);
3357 } else {
3358 if (picLevel == PICLevel::SmallPIC)
3359 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
3360 else
3361 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
3362 }
3363 return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT,
3364 GOTPtr, TGA, TGA);
3365 }
3366
3367 if (Model == TLSModel::LocalDynamic) {
3368 if (Subtarget.isUsingPCRelativeCalls()) {
3369 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3370 PPCII::MO_GOT_TLSLD_PCREL_FLAG);
3371 SDValue MatPCRel =
3372 DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA);
3373 return DAG.getNode(PPCISD::PADDI_DTPREL, dl, PtrVT, MatPCRel, TGA);
3374 }
3375
3376 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
3377 SDValue GOTPtr;
3378 if (is64bit) {
3379 setUsesTOCBasePtr(DAG);
3380 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
3381 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT,
3382 GOTReg, TGA);
3383 } else {
3384 if (picLevel == PICLevel::SmallPIC)
3385 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
3386 else
3387 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
3388 }
3389 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl,
3390 PtrVT, GOTPtr, TGA, TGA);
3391 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl,
3392 PtrVT, TLSAddr, TGA);
3393 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA);
3394 }
3395
3396 llvm_unreachable("Unknown TLS model!")::llvm::llvm_unreachable_internal("Unknown TLS model!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3396)
;
3397}
3398
3399SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
3400 SelectionDAG &DAG) const {
3401 EVT PtrVT = Op.getValueType();
3402 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op);
3403 SDLoc DL(GSDN);
3404 const GlobalValue *GV = GSDN->getGlobal();
3405
3406 // 64-bit SVR4 ABI & AIX ABI code is always position-independent.
3407 // The actual address of the GlobalValue is stored in the TOC.
3408 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3409 if (Subtarget.isUsingPCRelativeCalls()) {
3410 EVT Ty = getPointerTy(DAG.getDataLayout());
3411 if (isAccessedAsGotIndirect(Op)) {
3412 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(),
3413 PPCII::MO_PCREL_FLAG |
3414 PPCII::MO_GOT_FLAG);
3415 SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
3416 SDValue Load = DAG.getLoad(MVT::i64, DL, DAG.getEntryNode(), MatPCRel,
3417 MachinePointerInfo());
3418 return Load;
3419 } else {
3420 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(),
3421 PPCII::MO_PCREL_FLAG);
3422 return DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
3423 }
3424 }
3425 setUsesTOCBasePtr(DAG);
3426 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset());
3427 return getTOCEntry(DAG, DL, GA);
3428 }
3429
3430 unsigned MOHiFlag, MOLoFlag;
3431 bool IsPIC = isPositionIndependent();
3432 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV);
3433
3434 if (IsPIC && Subtarget.isSVR4ABI()) {
3435 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT,
3436 GSDN->getOffset(),
3437 PPCII::MO_PIC_FLAG);
3438 return getTOCEntry(DAG, DL, GA);
3439 }
3440
3441 SDValue GAHi =
3442 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag);
3443 SDValue GALo =
3444 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag);
3445
3446 return LowerLabelRef(GAHi, GALo, IsPIC, DAG);
3447}
3448
3449SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3450 bool IsStrict = Op->isStrictFPOpcode();
3451 ISD::CondCode CC =
3452 cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get();
3453 SDValue LHS = Op.getOperand(IsStrict ? 1 : 0);
3454 SDValue RHS = Op.getOperand(IsStrict ? 2 : 1);
3455 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
3456 EVT LHSVT = LHS.getValueType();
3457 SDLoc dl(Op);
3458
3459 // Soften the setcc with libcall if it is fp128.
3460 if (LHSVT == MVT::f128) {
3461 assert(!Subtarget.hasP9Vector() &&(static_cast <bool> (!Subtarget.hasP9Vector() &&
"SETCC for f128 is already legal under Power9!") ? void (0) :
__assert_fail ("!Subtarget.hasP9Vector() && \"SETCC for f128 is already legal under Power9!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3462, __extension__ __PRETTY_FUNCTION__))
3462 "SETCC for f128 is already legal under Power9!")(static_cast <bool> (!Subtarget.hasP9Vector() &&
"SETCC for f128 is already legal under Power9!") ? void (0) :
__assert_fail ("!Subtarget.hasP9Vector() && \"SETCC for f128 is already legal under Power9!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3462, __extension__ __PRETTY_FUNCTION__))
;
3463 softenSetCCOperands(DAG, LHSVT, LHS, RHS, CC, dl, LHS, RHS, Chain,
3464 Op->getOpcode() == ISD::STRICT_FSETCCS);
3465 if (RHS.getNode())
3466 LHS = DAG.getNode(ISD::SETCC, dl, Op.getValueType(), LHS, RHS,
3467 DAG.getCondCode(CC));
3468 if (IsStrict)
3469 return DAG.getMergeValues({LHS, Chain}, dl);
3470 return LHS;
3471 }
3472
3473 assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!")(static_cast <bool> (!IsStrict && "Don't know how to handle STRICT_FSETCC!"
) ? void (0) : __assert_fail ("!IsStrict && \"Don't know how to handle STRICT_FSETCC!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3473, __extension__ __PRETTY_FUNCTION__))
;
3474
3475 if (Op.getValueType() == MVT::v2i64) {
3476 // When the operands themselves are v2i64 values, we need to do something
3477 // special because VSX has no underlying comparison operations for these.
3478 if (LHS.getValueType() == MVT::v2i64) {
3479 // Equality can be handled by casting to the legal type for Altivec
3480 // comparisons, everything else needs to be expanded.
3481 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3482 return DAG.getNode(
3483 ISD::BITCAST, dl, MVT::v2i64,
3484 DAG.getSetCC(dl, MVT::v4i32,
3485 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS),
3486 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC));
3487 }
3488
3489 return SDValue();
3490 }
3491
3492 // We handle most of these in the usual way.
3493 return Op;
3494 }
3495
3496 // If we're comparing for equality to zero, expose the fact that this is
3497 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can
3498 // fold the new nodes.
3499 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
3500 return V;
3501
3502 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
3503 // Leave comparisons against 0 and -1 alone for now, since they're usually
3504 // optimized. FIXME: revisit this when we can custom lower all setcc
3505 // optimizations.
3506 if (C->isAllOnesValue() || C->isNullValue())
3507 return SDValue();
3508 }
3509
3510 // If we have an integer seteq/setne, turn it into a compare against zero
3511 // by xor'ing the rhs with the lhs, which is faster than setting a
3512 // condition register, reading it back out, and masking the correct bit. The
3513 // normal approach here uses sub to do this instead of xor. Using xor exposes
3514 // the result to other bit-twiddling opportunities.
3515 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
3516 EVT VT = Op.getValueType();
3517 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, LHS, RHS);
3518 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC);
3519 }
3520 return SDValue();
3521}
3522
3523SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3524 SDNode *Node = Op.getNode();
3525 EVT VT = Node->getValueType(0);
3526 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3527 SDValue InChain = Node->getOperand(0);
3528 SDValue VAListPtr = Node->getOperand(1);
3529 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
3530 SDLoc dl(Node);
3531
3532 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only")(static_cast <bool> (!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"
) ? void (0) : __assert_fail ("!Subtarget.isPPC64() && \"LowerVAARG is PPC32 only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3532, __extension__ __PRETTY_FUNCTION__))
;
3533
3534 // gpr_index
3535 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
3536 VAListPtr, MachinePointerInfo(SV), MVT::i8);
3537 InChain = GprIndex.getValue(1);
3538
3539 if (VT == MVT::i64) {
3540 // Check if GprIndex is even
3541 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex,
3542 DAG.getConstant(1, dl, MVT::i32));
3543 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd,
3544 DAG.getConstant(0, dl, MVT::i32), ISD::SETNE);
3545 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex,
3546 DAG.getConstant(1, dl, MVT::i32));
3547 // Align GprIndex to be even if it isn't
3548 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne,
3549 GprIndex);
3550 }
3551
3552 // fpr index is 1 byte after gpr
3553 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
3554 DAG.getConstant(1, dl, MVT::i32));
3555
3556 // fpr
3557 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
3558 FprPtr, MachinePointerInfo(SV), MVT::i8);
3559 InChain = FprIndex.getValue(1);
3560
3561 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
3562 DAG.getConstant(8, dl, MVT::i32));
3563
3564 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
3565 DAG.getConstant(4, dl, MVT::i32));
3566
3567 // areas
3568 SDValue OverflowArea =
3569 DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo());
3570 InChain = OverflowArea.getValue(1);
3571
3572 SDValue RegSaveArea =
3573 DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo());
3574 InChain = RegSaveArea.getValue(1);
3575
3576 // select overflow_area if index > 8
3577 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex,
3578 DAG.getConstant(8, dl, MVT::i32), ISD::SETLT);
3579
3580 // adjustment constant gpr_index * 4/8
3581 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32,
3582 VT.isInteger() ? GprIndex : FprIndex,
3583 DAG.getConstant(VT.isInteger() ? 4 : 8, dl,
3584 MVT::i32));
3585
3586 // OurReg = RegSaveArea + RegConstant
3587 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea,
3588 RegConstant);
3589
3590 // Floating types are 32 bytes into RegSaveArea
3591 if (VT.isFloatingPoint())
3592 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg,
3593 DAG.getConstant(32, dl, MVT::i32));
3594
3595 // increase {f,g}pr_index by 1 (or 2 if VT is i64)
3596 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32,
3597 VT.isInteger() ? GprIndex : FprIndex,
3598 DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl,
3599 MVT::i32));
3600
3601 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1,
3602 VT.isInteger() ? VAListPtr : FprPtr,
3603 MachinePointerInfo(SV), MVT::i8);
3604
3605 // determine if we should load from reg_save_area or overflow_area
3606 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea);
3607
3608 // increase overflow_area by 4/8 if gpr/fpr > 8
3609 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea,
3610 DAG.getConstant(VT.isInteger() ? 4 : 8,
3611 dl, MVT::i32));
3612
3613 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea,
3614 OverflowAreaPlusN);
3615
3616 InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr,
3617 MachinePointerInfo(), MVT::i32);
3618
3619 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo());
3620}
3621
3622SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3623 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only")(static_cast <bool> (!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"
) ? void (0) : __assert_fail ("!Subtarget.isPPC64() && \"LowerVACOPY is PPC32 only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3623, __extension__ __PRETTY_FUNCTION__))
;
3624
3625 // We have to copy the entire va_list struct:
3626 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
3627 return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2),
3628 DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8),
3629 false, true, false, MachinePointerInfo(),
3630 MachinePointerInfo());
3631}
3632
3633SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3634 SelectionDAG &DAG) const {
3635 if (Subtarget.isAIXABI())
3636 report_fatal_error("ADJUST_TRAMPOLINE operation is not supported on AIX.");
3637
3638 return Op.getOperand(0);
3639}
3640
3641SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
3642 MachineFunction &MF = DAG.getMachineFunction();
3643 PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>();
3644
3645 assert((Op.getOpcode() == ISD::INLINEASM ||(static_cast <bool> ((Op.getOpcode() == ISD::INLINEASM ||
Op.getOpcode() == ISD::INLINEASM_BR) && "Expecting Inline ASM node."
) ? void (0) : __assert_fail ("(Op.getOpcode() == ISD::INLINEASM || Op.getOpcode() == ISD::INLINEASM_BR) && \"Expecting Inline ASM node.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3647, __extension__ __PRETTY_FUNCTION__))
3646 Op.getOpcode() == ISD::INLINEASM_BR) &&(static_cast <bool> ((Op.getOpcode() == ISD::INLINEASM ||
Op.getOpcode() == ISD::INLINEASM_BR) && "Expecting Inline ASM node."
) ? void (0) : __assert_fail ("(Op.getOpcode() == ISD::INLINEASM || Op.getOpcode() == ISD::INLINEASM_BR) && \"Expecting Inline ASM node.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3647, __extension__ __PRETTY_FUNCTION__))
3647 "Expecting Inline ASM node.")(static_cast <bool> ((Op.getOpcode() == ISD::INLINEASM ||
Op.getOpcode() == ISD::INLINEASM_BR) && "Expecting Inline ASM node."
) ? void (0) : __assert_fail ("(Op.getOpcode() == ISD::INLINEASM || Op.getOpcode() == ISD::INLINEASM_BR) && \"Expecting Inline ASM node.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3647, __extension__ __PRETTY_FUNCTION__))
;
3648
3649 // If an LR store is already known to be required then there is not point in
3650 // checking this ASM as well.
3651 if (MFI.isLRStoreRequired())
3652 return Op;
3653
3654 // Inline ASM nodes have an optional last operand that is an incoming Flag of
3655 // type MVT::Glue. We want to ignore this last operand if that is the case.
3656 unsigned NumOps = Op.getNumOperands();
3657 if (Op.getOperand(NumOps - 1).getValueType() == MVT::Glue)
3658 --NumOps;
3659
3660 // Check all operands that may contain the LR.
3661 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
3662 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
3663 unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
3664 ++i; // Skip the ID value.
3665
3666 switch (InlineAsm::getKind(Flags)) {
3667 default:
3668 llvm_unreachable("Bad flags!")::llvm::llvm_unreachable_internal("Bad flags!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3668)
;
3669 case InlineAsm::Kind_RegUse:
3670 case InlineAsm::Kind_Imm:
3671 case InlineAsm::Kind_Mem:
3672 i += NumVals;
3673 break;
3674 case InlineAsm::Kind_Clobber:
3675 case InlineAsm::Kind_RegDef:
3676 case InlineAsm::Kind_RegDefEarlyClobber: {
3677 for (; NumVals; --NumVals, ++i) {
3678 Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
3679 if (Reg != PPC::LR && Reg != PPC::LR8)
3680 continue;
3681 MFI.setLRStoreRequired();
3682 return Op;
3683 }
3684 break;
3685 }
3686 }
3687 }
3688
3689 return Op;
3690}
3691
3692SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3693 SelectionDAG &DAG) const {
3694 if (Subtarget.isAIXABI())
3695 report_fatal_error("INIT_TRAMPOLINE operation is not supported on AIX.");
3696
3697 SDValue Chain = Op.getOperand(0);
3698 SDValue Trmp = Op.getOperand(1); // trampoline
3699 SDValue FPtr = Op.getOperand(2); // nested function
3700 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
3701 SDLoc dl(Op);
3702
3703 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3704 bool isPPC64 = (PtrVT == MVT::i64);
3705 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
3706
3707 TargetLowering::ArgListTy Args;
3708 TargetLowering::ArgListEntry Entry;
3709
3710 Entry.Ty = IntPtrTy;
3711 Entry.Node = Trmp; Args.push_back(Entry);
3712
3713 // TrampSize == (isPPC64 ? 48 : 40);
3714 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl,
3715 isPPC64 ? MVT::i64 : MVT::i32);
3716 Args.push_back(Entry);
3717
3718 Entry.Node = FPtr; Args.push_back(Entry);
3719 Entry.Node = Nest; Args.push_back(Entry);
3720
3721 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
3722 TargetLowering::CallLoweringInfo CLI(DAG);
3723 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3724 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3725 DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args));
3726
3727 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3728 return CallResult.second;
3729}
3730
3731SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3732 MachineFunction &MF = DAG.getMachineFunction();
3733 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3734 EVT PtrVT = getPointerTy(MF.getDataLayout());
3735
3736 SDLoc dl(Op);
3737
3738 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) {
3739 // vastart just stores the address of the VarArgsFrameIndex slot into the
3740 // memory location argument.
3741 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
3742 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3743 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
3744 MachinePointerInfo(SV));
3745 }
3746
3747 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
3748 // We suppose the given va_list is already allocated.
3749 //
3750 // typedef struct {
3751 // char gpr; /* index into the array of 8 GPRs
3752 // * stored in the register save area
3753 // * gpr=0 corresponds to r3,
3754 // * gpr=1 to r4, etc.
3755 // */
3756 // char fpr; /* index into the array of 8 FPRs
3757 // * stored in the register save area
3758 // * fpr=0 corresponds to f1,
3759 // * fpr=1 to f2, etc.
3760 // */
3761 // char *overflow_arg_area;
3762 // /* location on stack that holds
3763 // * the next overflow argument
3764 // */
3765 // char *reg_save_area;
3766 // /* where r3:r10 and f1:f8 (if saved)
3767 // * are stored
3768 // */
3769 // } va_list[1];
3770
3771 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32);
3772 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32);
3773 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(),
3774 PtrVT);
3775 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3776 PtrVT);
3777
3778 uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
3779 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT);
3780
3781 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
3782 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT);
3783
3784 uint64_t FPROffset = 1;
3785 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT);
3786
3787 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3788
3789 // Store first byte : number of int regs
3790 SDValue firstStore =
3791 DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1),
3792 MachinePointerInfo(SV), MVT::i8);
3793 uint64_t nextOffset = FPROffset;
3794 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1),
3795 ConstFPROffset);
3796
3797 // Store second byte : number of float regs
3798 SDValue secondStore =
3799 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr,
3800 MachinePointerInfo(SV, nextOffset), MVT::i8);
3801 nextOffset += StackOffset;
3802 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset);
3803
3804 // Store second word : arguments given on stack
3805 SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr,
3806 MachinePointerInfo(SV, nextOffset));
3807 nextOffset += FrameOffset;
3808 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset);
3809
3810 // Store third word : arguments given in registers
3811 return DAG.getStore(thirdStore, dl, FR, nextPtr,
3812 MachinePointerInfo(SV, nextOffset));
3813}
3814
3815/// FPR - The set of FP registers that should be allocated for arguments
3816/// on Darwin and AIX.
3817static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5,
3818 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10,
3819 PPC::F11, PPC::F12, PPC::F13};
3820
3821/// CalculateStackSlotSize - Calculates the size reserved for this argument on
3822/// the stack.
3823static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
3824 unsigned PtrByteSize) {
3825 unsigned ArgSize = ArgVT.getStoreSize();
3826 if (Flags.isByVal())
3827 ArgSize = Flags.getByValSize();
3828
3829 // Round up to multiples of the pointer size, except for array members,
3830 // which are always packed.
3831 if (!Flags.isInConsecutiveRegs())
3832 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
3833
3834 return ArgSize;
3835}
3836
3837/// CalculateStackSlotAlignment - Calculates the alignment of this argument
3838/// on the stack.
3839static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
3840 ISD::ArgFlagsTy Flags,
3841 unsigned PtrByteSize) {
3842 Align Alignment(PtrByteSize);
3843
3844 // Altivec parameters are padded to a 16 byte boundary.
3845 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
3846 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
3847 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
3848 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
3849 Alignment = Align(16);
3850
3851 // ByVal parameters are aligned as requested.
3852 if (Flags.isByVal()) {
3853 auto BVAlign = Flags.getNonZeroByValAlign();
3854 if (BVAlign > PtrByteSize) {
3855 if (BVAlign.value() % PtrByteSize != 0)
3856 llvm_unreachable(::llvm::llvm_unreachable_internal("ByVal alignment is not a multiple of the pointer size"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3857)
3857 "ByVal alignment is not a multiple of the pointer size")::llvm::llvm_unreachable_internal("ByVal alignment is not a multiple of the pointer size"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3857)
;
3858
3859 Alignment = BVAlign;
3860 }
3861 }
3862
3863 // Array members are always packed to their original alignment.
3864 if (Flags.isInConsecutiveRegs()) {
3865 // If the array member was split into multiple registers, the first
3866 // needs to be aligned to the size of the full type. (Except for
3867 // ppcf128, which is only aligned as its f64 components.)
3868 if (Flags.isSplit() && OrigVT != MVT::ppcf128)
3869 Alignment = Align(OrigVT.getStoreSize());
3870 else
3871 Alignment = Align(ArgVT.getStoreSize());
3872 }
3873
3874 return Alignment;
3875}
3876
3877/// CalculateStackSlotUsed - Return whether this argument will use its
3878/// stack slot (instead of being passed in registers). ArgOffset,
3879/// AvailableFPRs, and AvailableVRs must hold the current argument
3880/// position, and will be updated to account for this argument.
3881static bool CalculateStackSlotUsed(const PPCSubtarget &Subtarget, EVT ArgVT,
3882 EVT OrigVT, ISD::ArgFlagsTy Flags,
3883 unsigned PtrByteSize, unsigned LinkageSize,
3884 unsigned ParamAreaSize, unsigned &ArgOffset,
3885 unsigned &AvailableFPRs,
3886 unsigned &AvailableVRs) {
3887 bool UseMemory = false;
3888
3889 // Respect alignment of argument on the stack.
3890 Align Alignment =
3891 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
3892 ArgOffset = alignTo(ArgOffset, Alignment);
3893 // If there's no space left in the argument save area, we must
3894 // use memory (this check also catches zero-sized arguments).
3895 if (ArgOffset >= LinkageSize + ParamAreaSize)
3896 UseMemory = true;
3897
3898 // Allocate argument on the stack.
3899 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
3900 if (Flags.isInConsecutiveRegsLast())
3901 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
3902 // If we overran the argument save area, we must use memory
3903 // (this check catches arguments passed partially in memory)
3904 if (ArgOffset > LinkageSize + ParamAreaSize)
3905 UseMemory = true;
3906
3907 // However, if the argument is actually passed in an FPR or a VR,
3908 // we don't use memory after all.
3909 if (!Flags.isByVal()) {
3910 if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
3911 if (AvailableFPRs > 0) {
3912 --AvailableFPRs;
3913 return false;
3914 }
3915 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
3916 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
3917 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
3918 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
3919 if (AvailableVRs > 0) {
3920 --AvailableVRs;
3921 return false;
3922 }
3923 } else if (Subtarget.isPPC64() && Subtarget.isELFv2ABI() &&
3924 Flags.getByValSize() >= 8)
3925 // For 64-bit ELF v2, passing by value object whose size is no less than 8
3926 // bytes will be copied to parameter save area. This is for compatibility
3927 // for other compiler which requires byval parameters to be stored in
3928 // caller's parameter save area.
3929 return true;
3930
3931 return UseMemory;
3932}
3933
3934/// EnsureStackAlignment - Round stack frame size up from NumBytes to
3935/// ensure minimum alignment required for target.
3936static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
3937 unsigned NumBytes) {
3938 return alignTo(NumBytes, Lowering->getStackAlign());
3939}
3940
3941SDValue PPCTargetLowering::LowerFormalArguments(
3942 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3943 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
3944 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3945 if (Subtarget.isAIXABI())
3946 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG,
3947 InVals);
3948 if (Subtarget.is64BitELFABI())
3949 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
3950 InVals);
3951 assert(Subtarget.is32BitELFABI())(static_cast <bool> (Subtarget.is32BitELFABI()) ? void (
0) : __assert_fail ("Subtarget.is32BitELFABI()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 3951, __extension__ __PRETTY_FUNCTION__))
;
3952 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
3953 InVals);
3954}
3955
3956SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
3957 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3958 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
3959 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3960
3961 // 32-bit SVR4 ABI Stack Frame Layout:
3962 // +-----------------------------------+
3963 // +--> | Back chain |
3964 // | +-----------------------------------+
3965 // | | Floating-point register save area |
3966 // | +-----------------------------------+
3967 // | | General register save area |
3968 // | +-----------------------------------+
3969 // | | CR save word |
3970 // | +-----------------------------------+
3971 // | | VRSAVE save word |
3972 // | +-----------------------------------+
3973 // | | Alignment padding |
3974 // | +-----------------------------------+
3975 // | | Vector register save area |
3976 // | +-----------------------------------+
3977 // | | Local variable space |
3978 // | +-----------------------------------+
3979 // | | Parameter list area |
3980 // | +-----------------------------------+
3981 // | | LR save word |
3982 // | +-----------------------------------+
3983 // SP--> +--- | Back chain |
3984 // +-----------------------------------+
3985 //
3986 // Specifications:
3987 // System V Application Binary Interface PowerPC Processor Supplement
3988 // AltiVec Technology Programming Interface Manual
3989
3990 MachineFunction &MF = DAG.getMachineFunction();
3991 MachineFrameInfo &MFI = MF.getFrameInfo();
3992 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3993
3994 EVT PtrVT = getPointerTy(MF.getDataLayout());
3995 // Potential tail calls could cause overwriting of argument stack slots.
3996 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
3997 (CallConv == CallingConv::Fast));
3998 const Align PtrAlign(4);
3999
4000 // Assign locations to all of the incoming arguments.
4001 SmallVector<CCValAssign, 16> ArgLocs;
4002 PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4003 *DAG.getContext());
4004
4005 // Reserve space for the linkage area on the stack.
4006 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4007 CCInfo.AllocateStack(LinkageSize, PtrAlign);
4008 if (useSoftFloat())
4009 CCInfo.PreAnalyzeFormalArguments(Ins);
4010
4011 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4);
4012 CCInfo.clearWasPPCF128();
4013
4014 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4015 CCValAssign &VA = ArgLocs[i];
4016
4017 // Arguments stored in registers.
4018 if (VA.isRegLoc()) {
4019 const TargetRegisterClass *RC;
4020 EVT ValVT = VA.getValVT();
4021
4022 switch (ValVT.getSimpleVT().SimpleTy) {
4023 default:
4024 llvm_unreachable("ValVT not supported by formal arguments Lowering")::llvm::llvm_unreachable_internal("ValVT not supported by formal arguments Lowering"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4024)
;
4025 case MVT::i1:
4026 case MVT::i32:
4027 RC = &PPC::GPRCRegClass;
4028 break;
4029 case MVT::f32:
4030 if (Subtarget.hasP8Vector())
4031 RC = &PPC::VSSRCRegClass;
4032 else if (Subtarget.hasSPE())
4033 RC = &PPC::GPRCRegClass;
4034 else
4035 RC = &PPC::F4RCRegClass;
4036 break;
4037 case MVT::f64:
4038 if (Subtarget.hasVSX())
4039 RC = &PPC::VSFRCRegClass;
4040 else if (Subtarget.hasSPE())
4041 // SPE passes doubles in GPR pairs.
4042 RC = &PPC::GPRCRegClass;
4043 else
4044 RC = &PPC::F8RCRegClass;
4045 break;
4046 case MVT::v16i8:
4047 case MVT::v8i16:
4048 case MVT::v4i32:
4049 RC = &PPC::VRRCRegClass;
4050 break;
4051 case MVT::v4f32:
4052 RC = &PPC::VRRCRegClass;
4053 break;
4054 case MVT::v2f64:
4055 case MVT::v2i64:
4056 RC = &PPC::VRRCRegClass;
4057 break;
4058 }
4059
4060 SDValue ArgValue;
4061 // Transform the arguments stored in physical registers into
4062 // virtual ones.
4063 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) {
4064 assert(i + 1 < e && "No second half of double precision argument")(static_cast <bool> (i + 1 < e && "No second half of double precision argument"
) ? void (0) : __assert_fail ("i + 1 < e && \"No second half of double precision argument\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4064, __extension__ __PRETTY_FUNCTION__))
;
4065 unsigned RegLo = MF.addLiveIn(VA.getLocReg(), RC);
4066 unsigned RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC);
4067 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32);
4068 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32);
4069 if (!Subtarget.isLittleEndian())
4070 std::swap (ArgValueLo, ArgValueHi);
4071 ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo,
4072 ArgValueHi);
4073 } else {
4074 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
4075 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
4076 ValVT == MVT::i1 ? MVT::i32 : ValVT);
4077 if (ValVT == MVT::i1)
4078 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue);
4079 }
4080
4081 InVals.push_back(ArgValue);
4082 } else {
4083 // Argument stored in memory.
4084 assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4084, __extension__ __PRETTY_FUNCTION__))
;
4085
4086 // Get the extended size of the argument type in stack
4087 unsigned ArgSize = VA.getLocVT().getStoreSize();
4088 // Get the actual size of the argument type
4089 unsigned ObjSize = VA.getValVT().getStoreSize();
4090 unsigned ArgOffset = VA.getLocMemOffset();
4091 // Stack objects in PPC32 are right justified.
4092 ArgOffset += ArgSize - ObjSize;
4093 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable);
4094
4095 // Create load nodes to retrieve arguments from the stack.
4096 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4097 InVals.push_back(
4098 DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo()));
4099 }
4100 }
4101
4102 // Assign locations to all of the incoming aggregate by value arguments.
4103 // Aggregates passed by value are stored in the local variable space of the
4104 // caller's stack frame, right above the parameter list area.
4105 SmallVector<CCValAssign, 16> ByValArgLocs;
4106 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
4107 ByValArgLocs, *DAG.getContext());
4108
4109 // Reserve stack space for the allocations in CCInfo.
4110 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrAlign);
4111
4112 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal);
4113
4114 // Area that is at least reserved in the caller of this function.
4115 unsigned MinReservedArea = CCByValInfo.getNextStackOffset();
4116 MinReservedArea = std::max(MinReservedArea, LinkageSize);
4117
4118 // Set the size that is at least reserved in caller of this function. Tail
4119 // call optimized function's reserved stack space needs to be aligned so that
4120 // taking the difference between two stack areas will result in an aligned
4121 // stack.
4122 MinReservedArea =
4123 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
4124 FuncInfo->setMinReservedArea(MinReservedArea);
4125
4126 SmallVector<SDValue, 8> MemOps;
4127
4128 // If the function takes variable number of arguments, make a frame index for
4129 // the start of the first vararg value... for expansion of llvm.va_start.
4130 if (isVarArg) {
4131 static const MCPhysReg GPArgRegs[] = {
4132 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
4133 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
4134 };
4135 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs);
4136
4137 static const MCPhysReg FPArgRegs[] = {
4138 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
4139 PPC::F8
4140 };
4141 unsigned NumFPArgRegs = array_lengthof(FPArgRegs);
4142
4143 if (useSoftFloat() || hasSPE())
4144 NumFPArgRegs = 0;
4145
4146 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs));
4147 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs));
4148
4149 // Make room for NumGPArgRegs and NumFPArgRegs.
4150 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
4151 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
4152
4153 FuncInfo->setVarArgsStackOffset(
4154 MFI.CreateFixedObject(PtrVT.getSizeInBits()/8,
4155 CCInfo.getNextStackOffset(), true));
4156
4157 FuncInfo->setVarArgsFrameIndex(
4158 MFI.CreateStackObject(Depth, Align(8), false));
4159 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4160
4161 // The fixed integer arguments of a variadic function are stored to the
4162 // VarArgsFrameIndex on the stack so that they may be loaded by
4163 // dereferencing the result of va_next.
4164 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) {
4165 // Get an existing live-in vreg, or add a new one.
4166 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]);
4167 if (!VReg)
4168 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass);
4169
4170 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4171 SDValue Store =
4172 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
4173 MemOps.push_back(Store);
4174 // Increment the address by four for the next argument to store
4175 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
4176 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
4177 }
4178
4179 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
4180 // is set.
4181 // The double arguments are stored to the VarArgsFrameIndex
4182 // on the stack.
4183 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
4184 // Get an existing live-in vreg, or add a new one.
4185 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]);
4186 if (!VReg)
4187 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass);
4188
4189 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64);
4190 SDValue Store =
4191 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
4192 MemOps.push_back(Store);
4193 // Increment the address by eight for the next argument to store
4194 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl,
4195 PtrVT);
4196 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
4197 }
4198 }
4199
4200 if (!MemOps.empty())
4201 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
4202
4203 return Chain;
4204}
4205
4206// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4207// value to MVT::i64 and then truncate to the correct register size.
4208SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
4209 EVT ObjectVT, SelectionDAG &DAG,
4210 SDValue ArgVal,
4211 const SDLoc &dl) const {
4212 if (Flags.isSExt())
4213 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal,
4214 DAG.getValueType(ObjectVT));
4215 else if (Flags.isZExt())
4216 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal,
4217 DAG.getValueType(ObjectVT));
4218
4219 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal);
4220}
4221
4222SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
4223 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4224 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4225 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4226 // TODO: add description of PPC stack frame format, or at least some docs.
4227 //
4228 bool isELFv2ABI = Subtarget.isELFv2ABI();
4229 bool isLittleEndian = Subtarget.isLittleEndian();
4230 MachineFunction &MF = DAG.getMachineFunction();
4231 MachineFrameInfo &MFI = MF.getFrameInfo();
4232 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4233
4234 assert(!(CallConv == CallingConv::Fast && isVarArg) &&(static_cast <bool> (!(CallConv == CallingConv::Fast &&
isVarArg) && "fastcc not supported on varargs functions"
) ? void (0) : __assert_fail ("!(CallConv == CallingConv::Fast && isVarArg) && \"fastcc not supported on varargs functions\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4235, __extension__ __PRETTY_FUNCTION__))
4235 "fastcc not supported on varargs functions")(static_cast <bool> (!(CallConv == CallingConv::Fast &&
isVarArg) && "fastcc not supported on varargs functions"
) ? void (0) : __assert_fail ("!(CallConv == CallingConv::Fast && isVarArg) && \"fastcc not supported on varargs functions\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4235, __extension__ __PRETTY_FUNCTION__))
;
4236
4237 EVT PtrVT = getPointerTy(MF.getDataLayout());
4238 // Potential tail calls could cause overwriting of argument stack slots.
4239 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4240 (CallConv == CallingConv::Fast));
4241 unsigned PtrByteSize = 8;
4242 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4243
4244 static const MCPhysReg GPR[] = {
4245 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4246 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4247 };
4248 static const MCPhysReg VR[] = {
4249 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4250 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4251 };
4252
4253 const unsigned Num_GPR_Regs = array_lengthof(GPR);
4254 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
4255 const unsigned Num_VR_Regs = array_lengthof(VR);
4256
4257 // Do a first pass over the arguments to determine whether the ABI
4258 // guarantees that our caller has allocated the parameter save area
4259 // on its stack frame. In the ELFv1 ABI, this is always the case;
4260 // in the ELFv2 ABI, it is true if this is a vararg function or if
4261 // any parameter is located in a stack slot.
4262
4263 bool HasParameterArea = !isELFv2ABI || isVarArg;
4264 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
4265 unsigned NumBytes = LinkageSize;
4266 unsigned AvailableFPRs = Num_FPR_Regs;
4267 unsigned AvailableVRs = Num_VR_Regs;
4268 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
4269 if (Ins[i].Flags.isNest())
4270 continue;
4271
4272 if (CalculateStackSlotUsed(Subtarget, Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
4273 PtrByteSize, LinkageSize, ParamAreaSize,
4274 NumBytes, AvailableFPRs, AvailableVRs))
4275 HasParameterArea = true;
4276 }
4277
4278 // Add DAG nodes to load the arguments or copy them out of registers. On
4279 // entry to a function on PPC, the arguments start after the linkage area,
4280 // although the first ones are often in registers.
4281
4282 unsigned ArgOffset = LinkageSize;
4283 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
4284 SmallVector<SDValue, 8> MemOps;
4285 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
4286 unsigned CurArgIdx = 0;
4287 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
4288 SDValue ArgVal;
4289 bool needsLoad = false;
4290 EVT ObjectVT = Ins[ArgNo].VT;
4291 EVT OrigVT = Ins[ArgNo].ArgVT;
4292 unsigned ObjSize = ObjectVT.getStoreSize();
4293 unsigned ArgSize = ObjSize;
4294 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
4295 if (Ins[ArgNo].isOrigArg()) {
4296 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
4297 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
4298 }
4299 // We re-align the argument offset for each argument, except when using the
4300 // fast calling convention, when we need to make sure we do that only when
4301 // we'll actually use a stack slot.
4302 unsigned CurArgOffset;
4303 Align Alignment;
4304 auto ComputeArgOffset = [&]() {
4305 /* Respect alignment of argument on the stack. */
4306 Alignment =
4307 CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize);
4308 ArgOffset = alignTo(ArgOffset, Alignment);
4309 CurArgOffset = ArgOffset;
4310 };
4311
4312 if (CallConv != CallingConv::Fast) {
4313 ComputeArgOffset();
4314
4315 /* Compute GPR index associated with argument offset. */
4316 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4317 GPR_idx = std::min(GPR_idx, Num_GPR_Regs);
4318 }
4319
4320 // FIXME the codegen can be much improved in some cases.
4321 // We do not have to keep everything in memory.
4322 if (Flags.isByVal()) {
4323 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit")(static_cast <bool> (Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"
) ? void (0) : __assert_fail ("Ins[ArgNo].isOrigArg() && \"Byval arguments cannot be implicit\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4323, __extension__ __PRETTY_FUNCTION__))
;
4324
4325 if (CallConv == CallingConv::Fast)
4326 ComputeArgOffset();
4327
4328 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
4329 ObjSize = Flags.getByValSize();
4330 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4331 // Empty aggregate parameters do not take up registers. Examples:
4332 // struct { } a;
4333 // union { } b;
4334 // int c[0];
4335 // etc. However, we have to provide a place-holder in InVals, so
4336 // pretend we have an 8-byte item at the current address for that
4337 // purpose.
4338 if (!ObjSize) {
4339 int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true);
4340 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4341 InVals.push_back(FIN);
4342 continue;
4343 }
4344
4345 // Create a stack object covering all stack doublewords occupied
4346 // by the argument. If the argument is (fully or partially) on
4347 // the stack, or if the argument is fully in registers but the
4348 // caller has allocated the parameter save anyway, we can refer
4349 // directly to the caller's stack frame. Otherwise, create a
4350 // local copy in our own frame.
4351 int FI;
4352 if (HasParameterArea ||
4353 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
4354 FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true);
4355 else
4356 FI = MFI.CreateStackObject(ArgSize, Alignment, false);
4357 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4358
4359 // Handle aggregates smaller than 8 bytes.
4360 if (ObjSize < PtrByteSize) {
4361 // The value of the object is its address, which differs from the
4362 // address of the enclosing doubleword on big-endian systems.
4363 SDValue Arg = FIN;
4364 if (!isLittleEndian) {
4365 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT);
4366 Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff);
4367 }
4368 InVals.push_back(Arg);
4369
4370 if (GPR_idx != Num_GPR_Regs) {
4371 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
4372 FuncInfo->addLiveInAttr(VReg, Flags);
4373 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4374 SDValue Store;
4375
4376 if (ObjSize==1 || ObjSize==2 || ObjSize==4) {
4377 EVT ObjType = (ObjSize == 1 ? MVT::i8 :
4378 (ObjSize == 2 ? MVT::i16 : MVT::i32));
4379 Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg,
4380 MachinePointerInfo(&*FuncArg), ObjType);
4381 } else {
4382 // For sizes that don't fit a truncating store (3, 5, 6, 7),
4383 // store the whole register as-is to the parameter save area
4384 // slot.
4385 Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4386 MachinePointerInfo(&*FuncArg));
4387 }
4388
4389 MemOps.push_back(Store);
4390 }
4391 // Whether we copied from a register or not, advance the offset
4392 // into the parameter save area by a full doubleword.
4393 ArgOffset += PtrByteSize;
4394 continue;
4395 }
4396
4397 // The value of the object is its address, which is the address of
4398 // its first stack doubleword.
4399 InVals.push_back(FIN);
4400
4401 // Store whatever pieces of the object are in registers to memory.
4402 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
4403 if (GPR_idx == Num_GPR_Regs)
4404 break;
4405
4406 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
4407 FuncInfo->addLiveInAttr(VReg, Flags);
4408 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4409 SDValue Addr = FIN;
4410 if (j) {
4411 SDValue Off = DAG.getConstant(j, dl, PtrVT);
4412 Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off);
4413 }
4414 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr,
4415 MachinePointerInfo(&*FuncArg, j));
4416 MemOps.push_back(Store);
4417 ++GPR_idx;
4418 }
4419 ArgOffset += ArgSize;
4420 continue;
4421 }
4422
4423 switch (ObjectVT.getSimpleVT().SimpleTy) {
4424 default: llvm_unreachable("Unhandled argument type!")::llvm::llvm_unreachable_internal("Unhandled argument type!",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4424)
;
4425 case MVT::i1:
4426 case MVT::i32:
4427 case MVT::i64:
4428 if (Flags.isNest()) {
4429 // The 'nest' parameter, if any, is passed in R11.
4430 unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass);
4431 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
4432
4433 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4434 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4435
4436 break;
4437 }
4438
4439 // These can be scalar arguments or elements of an integer array type
4440 // passed directly. Clang may use those instead of "byval" aggregate
4441 // types to avoid forcing arguments to memory unnecessarily.
4442 if (GPR_idx != Num_GPR_Regs) {
4443 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
4444 FuncInfo->addLiveInAttr(VReg, Flags);
4445 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
4446
4447 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4448 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4449 // value to MVT::i64 and then truncate to the correct register size.
4450 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4451 } else {
4452 if (CallConv == CallingConv::Fast)
4453 ComputeArgOffset();
4454
4455 needsLoad = true;
4456 ArgSize = PtrByteSize;
4457 }
4458 if (CallConv != CallingConv::Fast || needsLoad)
4459 ArgOffset += 8;
4460 break;
4461
4462 case MVT::f32:
4463 case MVT::f64:
4464 // These can be scalar arguments or elements of a float array type
4465 // passed directly. The latter are used to implement ELFv2 homogenous
4466 // float aggregates.
4467 if (FPR_idx != Num_FPR_Regs) {
4468 unsigned VReg;
4469
4470 if (ObjectVT == MVT::f32)
4471 VReg = MF.addLiveIn(FPR[FPR_idx],
4472 Subtarget.hasP8Vector()
4473 ? &PPC::VSSRCRegClass
4474 : &PPC::F4RCRegClass);
4475 else
4476 VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX()
4477 ? &PPC::VSFRCRegClass
4478 : &PPC::F8RCRegClass);
4479
4480 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
4481 ++FPR_idx;
4482 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
4483 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
4484 // once we support fp <-> gpr moves.
4485
4486 // This can only ever happen in the presence of f32 array types,
4487 // since otherwise we never run out of FPRs before running out
4488 // of GPRs.
4489 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
4490 FuncInfo->addLiveInAttr(VReg, Flags);
4491 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
4492
4493 if (ObjectVT == MVT::f32) {
4494 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
4495 ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal,
4496 DAG.getConstant(32, dl, MVT::i32));
4497 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal);
4498 }
4499
4500 ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal);
4501 } else {
4502 if (CallConv == CallingConv::Fast)
4503 ComputeArgOffset();
4504
4505 needsLoad = true;
4506 }
4507
4508 // When passing an array of floats, the array occupies consecutive
4509 // space in the argument area; only round up to the next doubleword
4510 // at the end of the array. Otherwise, each float takes 8 bytes.
4511 if (CallConv != CallingConv::Fast || needsLoad) {
4512 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
4513 ArgOffset += ArgSize;
4514 if (Flags.isInConsecutiveRegsLast())
4515 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4516 }
4517 break;
4518 case MVT::v4f32:
4519 case MVT::v4i32:
4520 case MVT::v8i16:
4521 case MVT::v16i8:
4522 case MVT::v2f64:
4523 case MVT::v2i64:
4524 case MVT::v1i128:
4525 case MVT::f128:
4526 // These can be scalar arguments or elements of a vector array type
4527 // passed directly. The latter are used to implement ELFv2 homogenous
4528 // vector aggregates.
4529 if (VR_idx != Num_VR_Regs) {
4530 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
4531 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
4532 ++VR_idx;
4533 } else {
4534 if (CallConv == CallingConv::Fast)
4535 ComputeArgOffset();
4536 needsLoad = true;
4537 }
4538 if (CallConv != CallingConv::Fast || needsLoad)
4539 ArgOffset += 16;
4540 break;
4541 }
4542
4543 // We need to load the argument to a virtual register if we determined
4544 // above that we ran out of physical registers of the appropriate type.
4545 if (needsLoad) {
4546 if (ObjSize < ArgSize && !isLittleEndian)
4547 CurArgOffset += ArgSize - ObjSize;
4548 int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable);
4549 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4550 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo());
4551 }
4552
4553 InVals.push_back(ArgVal);
4554 }
4555
4556 // Area that is at least reserved in the caller of this function.
4557 unsigned MinReservedArea;
4558 if (HasParameterArea)
4559 MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize);
4560 else
4561 MinReservedArea = LinkageSize;
4562
4563 // Set the size that is at least reserved in caller of this function. Tail
4564 // call optimized functions' reserved stack space needs to be aligned so that
4565 // taking the difference between two stack areas will result in an aligned
4566 // stack.
4567 MinReservedArea =
4568 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
4569 FuncInfo->setMinReservedArea(MinReservedArea);
4570
4571 // If the function takes variable number of arguments, make a frame index for
4572 // the start of the first vararg value... for expansion of llvm.va_start.
4573 // On ELFv2ABI spec, it writes:
4574 // C programs that are intended to be *portable* across different compilers
4575 // and architectures must use the header file <stdarg.h> to deal with variable
4576 // argument lists.
4577 if (isVarArg && MFI.hasVAStart()) {
4578 int Depth = ArgOffset;
4579
4580 FuncInfo->setVarArgsFrameIndex(
4581 MFI.CreateFixedObject(PtrByteSize, Depth, true));
4582 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4583
4584 // If this function is vararg, store any remaining integer argument regs
4585 // to their spots on the stack so that they may be loaded by dereferencing
4586 // the result of va_next.
4587 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4588 GPR_idx < Num_GPR_Regs; ++GPR_idx) {
4589 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
4590 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
4591 SDValue Store =
4592 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
4593 MemOps.push_back(Store);
4594 // Increment the address by four for the next argument to store
4595 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
4596 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
4597 }
4598 }
4599
4600 if (!MemOps.empty())
4601 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
4602
4603 return Chain;
4604}
4605
4606/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
4607/// adjusted to accommodate the arguments for the tailcall.
4608static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
4609 unsigned ParamSize) {
4610
4611 if (!isTailCall) return 0;
4612
4613 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
4614 unsigned CallerMinReservedArea = FI->getMinReservedArea();
4615 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
4616 // Remember only if the new adjustment is bigger.
4617 if (SPDiff < FI->getTailCallSPDelta())
4618 FI->setTailCallSPDelta(SPDiff);
4619
4620 return SPDiff;
4621}
4622
4623static bool isFunctionGlobalAddress(SDValue Callee);
4624
4625static bool callsShareTOCBase(const Function *Caller, SDValue Callee,
4626 const TargetMachine &TM) {
4627 // It does not make sense to call callsShareTOCBase() with a caller that
4628 // is PC Relative since PC Relative callers do not have a TOC.
4629#ifndef NDEBUG
4630 const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
4631 assert(!STICaller->isUsingPCRelativeCalls() &&(static_cast <bool> (!STICaller->isUsingPCRelativeCalls
() && "PC Relative callers do not have a TOC and cannot share a TOC Base"
) ? void (0) : __assert_fail ("!STICaller->isUsingPCRelativeCalls() && \"PC Relative callers do not have a TOC and cannot share a TOC Base\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4632, __extension__ __PRETTY_FUNCTION__))
4632 "PC Relative callers do not have a TOC and cannot share a TOC Base")(static_cast <bool> (!STICaller->isUsingPCRelativeCalls
() && "PC Relative callers do not have a TOC and cannot share a TOC Base"
) ? void (0) : __assert_fail ("!STICaller->isUsingPCRelativeCalls() && \"PC Relative callers do not have a TOC and cannot share a TOC Base\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4632, __extension__ __PRETTY_FUNCTION__))
;
4633#endif
4634
4635 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
4636 // don't have enough information to determine if the caller and callee share
4637 // the same TOC base, so we have to pessimistically assume they don't for
4638 // correctness.
4639 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
4640 if (!G)
4641 return false;
4642
4643 const GlobalValue *GV = G->getGlobal();
4644
4645 // If the callee is preemptable, then the static linker will use a plt-stub
4646 // which saves the toc to the stack, and needs a nop after the call
4647 // instruction to convert to a toc-restore.
4648 if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV))
4649 return false;
4650
4651 // Functions with PC Relative enabled may clobber the TOC in the same DSO.
4652 // We may need a TOC restore in the situation where the caller requires a
4653 // valid TOC but the callee is PC Relative and does not.
4654 const Function *F = dyn_cast<Function>(GV);
4655 const GlobalAlias *Alias = dyn_cast<GlobalAlias>(GV);
4656
4657 // If we have an Alias we can try to get the function from there.
4658 if (Alias) {
4659 const GlobalObject *GlobalObj = Alias->getBaseObject();
4660 F = dyn_cast<Function>(GlobalObj);
4661 }
4662
4663 // If we still have no valid function pointer we do not have enough
4664 // information to determine if the callee uses PC Relative calls so we must
4665 // assume that it does.
4666 if (!F)
4667 return false;
4668
4669 // If the callee uses PC Relative we cannot guarantee that the callee won't
4670 // clobber the TOC of the caller and so we must assume that the two
4671 // functions do not share a TOC base.
4672 const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(*F);
4673 if (STICallee->isUsingPCRelativeCalls())
4674 return false;
4675
4676 // If the GV is not a strong definition then we need to assume it can be
4677 // replaced by another function at link time. The function that replaces
4678 // it may not share the same TOC as the caller since the callee may be
4679 // replaced by a PC Relative version of the same function.
4680 if (!GV->isStrongDefinitionForLinker())
4681 return false;
4682
4683 // The medium and large code models are expected to provide a sufficiently
4684 // large TOC to provide all data addressing needs of a module with a
4685 // single TOC.
4686 if (CodeModel::Medium == TM.getCodeModel() ||
4687 CodeModel::Large == TM.getCodeModel())
4688 return true;
4689
4690 // Any explicitly-specified sections and section prefixes must also match.
4691 // Also, if we're using -ffunction-sections, then each function is always in
4692 // a different section (the same is true for COMDAT functions).
4693 if (TM.getFunctionSections() || GV->hasComdat() || Caller->hasComdat() ||
4694 GV->getSection() != Caller->getSection())
4695 return false;
4696 if (const auto *F = dyn_cast<Function>(GV)) {
4697 if (F->getSectionPrefix() != Caller->getSectionPrefix())
4698 return false;
4699 }
4700
4701 return true;
4702}
4703
4704static bool
4705needStackSlotPassParameters(const PPCSubtarget &Subtarget,
4706 const SmallVectorImpl<ISD::OutputArg> &Outs) {
4707 assert(Subtarget.is64BitELFABI())(static_cast <bool> (Subtarget.is64BitELFABI()) ? void (
0) : __assert_fail ("Subtarget.is64BitELFABI()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 4707, __extension__ __PRETTY_FUNCTION__))
;
4708
4709 const unsigned PtrByteSize = 8;
4710 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4711
4712 static const MCPhysReg GPR[] = {
4713 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4714 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4715 };
4716 static const MCPhysReg VR[] = {
4717 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4718 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4719 };
4720
4721 const unsigned NumGPRs = array_lengthof(GPR);
4722 const unsigned NumFPRs = 13;
4723 const unsigned NumVRs = array_lengthof(VR);
4724 const unsigned ParamAreaSize = NumGPRs * PtrByteSize;
4725
4726 unsigned NumBytes = LinkageSize;
4727 unsigned AvailableFPRs = NumFPRs;
4728 unsigned AvailableVRs = NumVRs;
4729
4730 for (const ISD::OutputArg& Param : Outs) {
4731 if (Param.Flags.isNest()) continue;
4732
4733 if (CalculateStackSlotUsed(Subtarget, Param.VT, Param.ArgVT, Param.Flags,
4734 PtrByteSize, LinkageSize, ParamAreaSize,
4735 NumBytes, AvailableFPRs, AvailableVRs))
4736 return true;
4737 }
4738 return false;
4739}
4740
4741static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
4742 if (CB.arg_size() != CallerFn->arg_size())
4743 return false;
4744
4745 auto CalleeArgIter = CB.arg_begin();
4746 auto CalleeArgEnd = CB.arg_end();
4747 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin();
4748
4749 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) {
4750 const Value* CalleeArg = *CalleeArgIter;
4751 const Value* CallerArg = &(*CallerArgIter);
4752 if (CalleeArg == CallerArg)
4753 continue;
4754
4755 // e.g. @caller([4 x i64] %a, [4 x i64] %b) {
4756 // tail call @callee([4 x i64] undef, [4 x i64] %b)
4757 // }
4758 // 1st argument of callee is undef and has the same type as caller.
4759 if (CalleeArg->getType() == CallerArg->getType() &&
4760 isa<UndefValue>(CalleeArg))
4761 continue;
4762
4763 return false;
4764 }
4765
4766 return true;
4767}
4768
4769// Returns true if TCO is possible between the callers and callees
4770// calling conventions.
4771static bool
4772areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
4773 CallingConv::ID CalleeCC) {
4774 // Tail calls are possible with fastcc and ccc.
4775 auto isTailCallableCC = [] (CallingConv::ID CC){
4776 return CC == CallingConv::C || CC == CallingConv::Fast;
4777 };
4778 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC))
4779 return false;
4780
4781 // We can safely tail call both fastcc and ccc callees from a c calling
4782 // convention caller. If the caller is fastcc, we may have less stack space
4783 // than a non-fastcc caller with the same signature so disable tail-calls in
4784 // that case.
4785 return CallerCC == CallingConv::C || CallerCC == CalleeCC;
4786}
4787
4788bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
4789 SDValue Callee, CallingConv::ID CalleeCC, const CallBase *CB, bool isVarArg,
4790 const SmallVectorImpl<ISD::OutputArg> &Outs,
4791 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4792 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
4793
4794 if (DisableSCO && !TailCallOpt) return false;
4795
4796 // Variadic argument functions are not supported.
4797 if (isVarArg) return false;
4798
4799 auto &Caller = DAG.getMachineFunction().getFunction();
4800 // Check that the calling conventions are compatible for tco.
4801 if (!areCallingConvEligibleForTCO_64SVR4(Caller.getCallingConv(), CalleeCC))
4802 return false;
4803
4804 // Caller contains any byval parameter is not supported.
4805 if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
4806 return false;
4807
4808 // Callee contains any byval parameter is not supported, too.
4809 // Note: This is a quick work around, because in some cases, e.g.
4810 // caller's stack size > callee's stack size, we are still able to apply
4811 // sibling call optimization. For example, gcc is able to do SCO for caller1
4812 // in the following example, but not for caller2.
4813 // struct test {
4814 // long int a;
4815 // char ary[56];
4816 // } gTest;
4817 // __attribute__((noinline)) int callee(struct test v, struct test *b) {
4818 // b->a = v.a;
4819 // return 0;
4820 // }
4821 // void caller1(struct test a, struct test c, struct test *b) {
4822 // callee(gTest, b); }
4823 // void caller2(struct test *b) { callee(gTest, b); }
4824 if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); }))
4825 return false;
4826
4827 // If callee and caller use different calling conventions, we cannot pass
4828 // parameters on stack since offsets for the parameter area may be different.
4829 if (Caller.getCallingConv() != CalleeCC &&
4830 needStackSlotPassParameters(Subtarget, Outs))
4831 return false;
4832
4833 // All variants of 64-bit ELF ABIs without PC-Relative addressing require that
4834 // the caller and callee share the same TOC for TCO/SCO. If the caller and
4835 // callee potentially have different TOC bases then we cannot tail call since
4836 // we need to restore the TOC pointer after the call.
4837 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977
4838 // We cannot guarantee this for indirect calls or calls to external functions.
4839 // When PC-Relative addressing is used, the concept of the TOC is no longer
4840 // applicable so this check is not required.
4841 // Check first for indirect calls.
4842 if (!Subtarget.isUsingPCRelativeCalls() &&
4843 !isFunctionGlobalAddress(Callee) && !isa<ExternalSymbolSDNode>(Callee))
4844 return false;
4845
4846 // Check if we share the TOC base.
4847 if (!Subtarget.isUsingPCRelativeCalls() &&
4848 !callsShareTOCBase(&Caller, Callee, getTargetMachine()))
4849 return false;
4850
4851 // TCO allows altering callee ABI, so we don't have to check further.
4852 if (CalleeCC == CallingConv::Fast && TailCallOpt)
4853 return true;
4854
4855 if (DisableSCO) return false;
4856
4857 // If callee use the same argument list that caller is using, then we can
4858 // apply SCO on this case. If it is not, then we need to check if callee needs
4859 // stack for passing arguments.
4860 // PC Relative tail calls may not have a CallBase.
4861 // If there is no CallBase we cannot verify if we have the same argument
4862 // list so assume that we don't have the same argument list.
4863 if (CB && !hasSameArgumentList(&Caller, *CB) &&
4864 needStackSlotPassParameters(Subtarget, Outs))
4865 return false;
4866 else if (!CB && needStackSlotPassParameters(Subtarget, Outs))
4867 return false;
4868
4869 return true;
4870}
4871
4872/// IsEligibleForTailCallOptimization - Check whether the call is eligible
4873/// for tail call optimization. Targets which want to do tail call
4874/// optimization should implement this function.
4875bool
4876PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
4877 CallingConv::ID CalleeCC,
4878 bool isVarArg,
4879 const SmallVectorImpl<ISD::InputArg> &Ins,
4880 SelectionDAG& DAG) const {
4881 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
4882 return false;
4883
4884 // Variable argument functions are not supported.
4885 if (isVarArg)
4886 return false;
4887
4888 MachineFunction &MF = DAG.getMachineFunction();
4889 CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
4890 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
4891 // Functions containing by val parameters are not supported.
4892 for (unsigned i = 0; i != Ins.size(); i++) {
4893 ISD::ArgFlagsTy Flags = Ins[i].Flags;
4894 if (Flags.isByVal()) return false;
4895 }
4896
4897 // Non-PIC/GOT tail calls are supported.
4898 if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
4899 return true;
4900
4901 // At the moment we can only do local tail calls (in same module, hidden
4902 // or protected) if we are generating PIC.
4903 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
4904 return G->getGlobal()->hasHiddenVisibility()
4905 || G->getGlobal()->hasProtectedVisibility();
4906 }
4907
4908 return false;
4909}
4910
4911/// isCallCompatibleAddress - Return the immediate to use if the specified
4912/// 32-bit value is representable in the immediate field of a BxA instruction.
4913static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
4914 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
4915 if (!C) return nullptr;
4916
4917 int Addr = C->getZExtValue();
4918 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
4919 SignExtend32<26>(Addr) != Addr)
4920 return nullptr; // Top 6 bits have to be sext of immediate.
4921
4922 return DAG
4923 .getConstant(
4924 (int)C->getZExtValue() >> 2, SDLoc(Op),
4925 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()))
4926 .getNode();
4927}
4928
4929namespace {
4930
4931struct TailCallArgumentInfo {
4932 SDValue Arg;
4933 SDValue FrameIdxOp;
4934 int FrameIdx = 0;
4935
4936 TailCallArgumentInfo() = default;
4937};
4938
4939} // end anonymous namespace
4940
4941/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
4942static void StoreTailCallArgumentsToStackSlot(
4943 SelectionDAG &DAG, SDValue Chain,
4944 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
4945 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
4946 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
4947 SDValue Arg = TailCallArgs[i].Arg;
4948 SDValue FIN = TailCallArgs[i].FrameIdxOp;
4949 int FI = TailCallArgs[i].FrameIdx;
4950 // Store relative to framepointer.
4951 MemOpChains.push_back(DAG.getStore(
4952 Chain, dl, Arg, FIN,
4953 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)));
4954 }
4955}
4956
4957/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
4958/// the appropriate stack slot for the tail call optimized function call.
4959static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
4960 SDValue OldRetAddr, SDValue OldFP,
4961 int SPDiff, const SDLoc &dl) {
4962 if (SPDiff) {
4963 // Calculate the new stack slot for the return address.
4964 MachineFunction &MF = DAG.getMachineFunction();
4965 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
4966 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
4967 bool isPPC64 = Subtarget.isPPC64();
4968 int SlotSize = isPPC64 ? 8 : 4;
4969 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
4970 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize,
4971 NewRetAddrLoc, true);
4972 EVT VT = isPPC64 ? MVT::i64 : MVT::i32;
4973 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT);
4974 Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx,
4975 MachinePointerInfo::getFixedStack(MF, NewRetAddr));
4976 }
4977 return Chain;
4978}
4979
4980/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
4981/// the position of the argument.
4982static void
4983CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64,
4984 SDValue Arg, int SPDiff, unsigned ArgOffset,
4985 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) {
4986 int Offset = ArgOffset + SPDiff;
4987 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8;
4988 int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
4989 EVT VT = isPPC64 ? MVT::i64 : MVT::i32;
4990 SDValue FIN = DAG.getFrameIndex(FI, VT);
4991 TailCallArgumentInfo Info;
4992 Info.Arg = Arg;
4993 Info.FrameIdxOp = FIN;
4994 Info.FrameIdx = FI;
4995 TailCallArguments.push_back(Info);
4996}
4997
4998/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
4999/// stack slot. Returns the chain as result and the loaded frame pointers in
5000/// LROpOut/FPOpout. Used when tail calling.
5001SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
5002 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
5003 SDValue &FPOpOut, const SDLoc &dl) const {
5004 if (SPDiff) {
5005 // Load the LR and FP stack slot for later adjusting.
5006 EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
5007 LROpOut = getReturnAddrFrameIndex(DAG);
5008 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo());
5009 Chain = SDValue(LROpOut.getNode(), 1);
5010 }
5011 return Chain;
5012}
5013
5014/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
5015/// by "Src" to address "Dst" of size "Size". Alignment information is
5016/// specified by the specific parameter attribute. The copy will be passed as
5017/// a byval function parameter.
5018/// Sometimes what we are copying is the end of a larger object, the part that
5019/// does not fit in registers.
5020static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
5021 SDValue Chain, ISD::ArgFlagsTy Flags,
5022 SelectionDAG &DAG, const SDLoc &dl) {
5023 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
5024 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode,
5025 Flags.getNonZeroByValAlign(), false, false, false,
5026 MachinePointerInfo(), MachinePointerInfo());
5027}
5028
5029/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
5030/// tail calls.
5031static void LowerMemOpCallTo(
5032 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
5033 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
5034 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
5035 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) {
5036 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
5037 if (!isTailCall) {
5038 if (isVector) {
5039 SDValue StackPtr;
5040 if (isPPC64)
5041 StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
5042 else
5043 StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
5044 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
5045 DAG.getConstant(ArgOffset, dl, PtrVT));
5046 }
5047 MemOpChains.push_back(
5048 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()));
5049 // Calculate and remember argument location.
5050 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset,
5051 TailCallArguments);
5052}
5053
5054static void
5055PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain,
5056 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
5057 SDValue FPOp,
5058 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5059 // Emit a sequence of copyto/copyfrom virtual registers for arguments that
5060 // might overwrite each other in case of tail call optimization.
5061 SmallVector<SDValue, 8> MemOpChains2;
5062 // Do not flag preceding copytoreg stuff together with the following stuff.
5063 InFlag = SDValue();
5064 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments,
5065 MemOpChains2, dl);
5066 if (!MemOpChains2.empty())
5067 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
5068
5069 // Store the return address to the appropriate stack slot.
5070 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl);
5071
5072 // Emit callseq_end just before tailcall node.
5073 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
5074 DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
5075 InFlag = Chain.getValue(1);
5076}
5077
5078// Is this global address that of a function that can be called by name? (as
5079// opposed to something that must hold a descriptor for an indirect call).
5080static bool isFunctionGlobalAddress(SDValue Callee) {
5081 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
5082 if (Callee.getOpcode() == ISD::GlobalTLSAddress ||
5083 Callee.getOpcode() == ISD::TargetGlobalTLSAddress)
5084 return false;
5085
5086 return G->getGlobal()->getValueType()->isFunctionTy();
5087 }
5088
5089 return false;
5090}
5091
5092SDValue PPCTargetLowering::LowerCallResult(
5093 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
5094 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5095 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
5096 SmallVector<CCValAssign, 16> RVLocs;
5097 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5098 *DAG.getContext());
5099
5100 CCRetInfo.AnalyzeCallResult(
5101 Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
5102 ? RetCC_PPC_Cold
5103 : RetCC_PPC);
5104
5105 // Copy all of the result registers out of their specified physreg.
5106 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
5107 CCValAssign &VA = RVLocs[i];
5108 assert(VA.isRegLoc() && "Can only return in registers!")(static_cast <bool> (VA.isRegLoc() && "Can only return in registers!"
) ? void (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5108, __extension__ __PRETTY_FUNCTION__))
;
5109
5110 SDValue Val;
5111
5112 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
5113 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
5114 InFlag);
5115 Chain = Lo.getValue(1);
5116 InFlag = Lo.getValue(2);
5117 VA = RVLocs[++i]; // skip ahead to next loc
5118 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
5119 InFlag);
5120 Chain = Hi.getValue(1);
5121 InFlag = Hi.getValue(2);
5122 if (!Subtarget.isLittleEndian())
5123 std::swap (Lo, Hi);
5124 Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi);
5125 } else {
5126 Val = DAG.getCopyFromReg(Chain, dl,
5127 VA.getLocReg(), VA.getLocVT(), InFlag);
5128 Chain = Val.getValue(1);
5129 InFlag = Val.getValue(2);
5130 }
5131
5132 switch (VA.getLocInfo()) {
5133 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5133)
;
5134 case CCValAssign::Full: break;
5135 case CCValAssign::AExt:
5136 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
5137 break;
5138 case CCValAssign::ZExt:
5139 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val,
5140 DAG.getValueType(VA.getValVT()));
5141 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
5142 break;
5143 case CCValAssign::SExt:
5144 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val,
5145 DAG.getValueType(VA.getValVT()));
5146 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
5147 break;
5148 }
5149
5150 InVals.push_back(Val);
5151 }
5152
5153 return Chain;
5154}
5155
5156static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG,
5157 const PPCSubtarget &Subtarget, bool isPatchPoint) {
5158 // PatchPoint calls are not indirect.
5159 if (isPatchPoint)
5160 return false;
5161
5162 if (isFunctionGlobalAddress(Callee) || isa<ExternalSymbolSDNode>(Callee))
5163 return false;
5164
5165 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not
5166 // becuase the immediate function pointer points to a descriptor instead of
5167 // a function entry point. The ELFv2 ABI cannot use a BLA because the function
5168 // pointer immediate points to the global entry point, while the BLA would
5169 // need to jump to the local entry point (see rL211174).
5170 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() &&
5171 isBLACompatibleAddress(Callee, DAG))
5172 return false;
5173
5174 return true;
5175}
5176
5177// AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls.
5178static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) {
5179 return Subtarget.isAIXABI() ||
5180 (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls());
5181}
5182
5183static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
5184 const Function &Caller,
5185 const SDValue &Callee,
5186 const PPCSubtarget &Subtarget,
5187 const TargetMachine &TM) {
5188 if (CFlags.IsTailCall)
5189 return PPCISD::TC_RETURN;
5190
5191 // This is a call through a function pointer.
5192 if (CFlags.IsIndirect) {
5193 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross
5194 // indirect calls. The save of the caller's TOC pointer to the stack will be
5195 // inserted into the DAG as part of call lowering. The restore of the TOC
5196 // pointer is modeled by using a pseudo instruction for the call opcode that
5197 // represents the 2 instruction sequence of an indirect branch and link,
5198 // immediately followed by a load of the TOC pointer from the the stack save
5199 // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC
5200 // as it is not saved or used.
5201 return isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC
5202 : PPCISD::BCTRL;
5203 }
5204
5205 if (Subtarget.isUsingPCRelativeCalls()) {
5206 assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI.")(static_cast <bool> (Subtarget.is64BitELFABI() &&
"PC Relative is only on ELF ABI.") ? void (0) : __assert_fail
("Subtarget.is64BitELFABI() && \"PC Relative is only on ELF ABI.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5206, __extension__ __PRETTY_FUNCTION__))
;
5207 return PPCISD::CALL_NOTOC;
5208 }
5209
5210 // The ABIs that maintain a TOC pointer accross calls need to have a nop
5211 // immediately following the call instruction if the caller and callee may
5212 // have different TOC bases. At link time if the linker determines the calls
5213 // may not share a TOC base, the call is redirected to a trampoline inserted
5214 // by the linker. The trampoline will (among other things) save the callers
5215 // TOC pointer at an ABI designated offset in the linkage area and the linker
5216 // will rewrite the nop to be a load of the TOC pointer from the linkage area
5217 // into gpr2.
5218 if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI())
5219 return callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL
5220 : PPCISD::CALL_NOP;
5221
5222 return PPCISD::CALL;
5223}
5224
5225static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
5226 const SDLoc &dl, const PPCSubtarget &Subtarget) {
5227 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI())
5228 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG))
5229 return SDValue(Dest, 0);
5230
5231 // Returns true if the callee is local, and false otherwise.
5232 auto isLocalCallee = [&]() {
5233 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
5234 const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
5235 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5236
5237 return DAG.getTarget().shouldAssumeDSOLocal(*Mod, GV) &&
5238 !dyn_cast_or_null<GlobalIFunc>(GV);
5239 };
5240
5241 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in
5242 // a static relocation model causes some versions of GNU LD (2.17.50, at
5243 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are
5244 // built with secure-PLT.
5245 bool UsePlt =
5246 Subtarget.is32BitELFABI() && !isLocalCallee() &&
5247 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
5248
5249 const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
5250 const TargetMachine &TM = Subtarget.getTargetMachine();
5251 const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
5252 MCSymbolXCOFF *S =
5253 cast<MCSymbolXCOFF>(TLOF->getFunctionEntryPointSymbol(GV, TM));
5254
5255 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
5256 return DAG.getMCSymbol(S, PtrVT);
5257 };
5258
5259 if (isFunctionGlobalAddress(Callee)) {
5260 const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
5261
5262 if (Subtarget.isAIXABI()) {
5263 assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX.")(static_cast <bool> (!isa<GlobalIFunc>(GV) &&
"IFunc is not supported on AIX.") ? void (0) : __assert_fail
("!isa<GlobalIFunc>(GV) && \"IFunc is not supported on AIX.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5263, __extension__ __PRETTY_FUNCTION__))
;
5264 return getAIXFuncEntryPointSymbolSDNode(GV);
5265 }
5266 return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0,
5267 UsePlt ? PPCII::MO_PLT : 0);
5268 }
5269
5270 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
5271 const char *SymName = S->getSymbol();
5272 if (Subtarget.isAIXABI()) {
5273 // If there exists a user-declared function whose name is the same as the
5274 // ExternalSymbol's, then we pick up the user-declared version.
5275 const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
5276 if (const Function *F =
5277 dyn_cast_or_null<Function>(Mod->getNamedValue(SymName)))
5278 return getAIXFuncEntryPointSymbolSDNode(F);
5279
5280 // On AIX, direct function calls reference the symbol for the function's
5281 // entry point, which is named by prepending a "." before the function's
5282 // C-linkage name. A Qualname is returned here because an external
5283 // function entry point is a csect with XTY_ER property.
5284 const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
5285 auto &Context = DAG.getMachineFunction().getMMI().getContext();
5286 MCSectionXCOFF *Sec = Context.getXCOFFSection(
5287 (Twine(".") + Twine(SymName)).str(), SectionKind::getMetadata(),
5288 XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
5289 return Sec->getQualNameSymbol();
5290 };
5291
5292 SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data();
5293 }
5294 return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(),
5295 UsePlt ? PPCII::MO_PLT : 0);
5296 }
5297
5298 // No transformation needed.
5299 assert(Callee.getNode() && "What no callee?")(static_cast <bool> (Callee.getNode() && "What no callee?"
) ? void (0) : __assert_fail ("Callee.getNode() && \"What no callee?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5299, __extension__ __PRETTY_FUNCTION__))
;
5300 return Callee;
5301}
5302
5303static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) {
5304 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START &&(static_cast <bool> (CallSeqStart.getOpcode() == ISD::CALLSEQ_START
&& "Expected a CALLSEQ_STARTSDNode.") ? void (0) : __assert_fail
("CallSeqStart.getOpcode() == ISD::CALLSEQ_START && \"Expected a CALLSEQ_STARTSDNode.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5305, __extension__ __PRETTY_FUNCTION__))
5305 "Expected a CALLSEQ_STARTSDNode.")(static_cast <bool> (CallSeqStart.getOpcode() == ISD::CALLSEQ_START
&& "Expected a CALLSEQ_STARTSDNode.") ? void (0) : __assert_fail
("CallSeqStart.getOpcode() == ISD::CALLSEQ_START && \"Expected a CALLSEQ_STARTSDNode.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5305, __extension__ __PRETTY_FUNCTION__))
;
5306
5307 // The last operand is the chain, except when the node has glue. If the node
5308 // has glue, then the last operand is the glue, and the chain is the second
5309 // last operand.
5310 SDValue LastValue = CallSeqStart.getValue(CallSeqStart->getNumValues() - 1);
5311 if (LastValue.getValueType() != MVT::Glue)
5312 return LastValue;
5313
5314 return CallSeqStart.getValue(CallSeqStart->getNumValues() - 2);
5315}
5316
5317// Creates the node that moves a functions address into the count register
5318// to prepare for an indirect call instruction.
5319static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5320 SDValue &Glue, SDValue &Chain,
5321 const SDLoc &dl) {
5322 SDValue MTCTROps[] = {Chain, Callee, Glue};
5323 EVT ReturnTypes[] = {MVT::Other, MVT::Glue};
5324 Chain = DAG.getNode(PPCISD::MTCTR, dl, makeArrayRef(ReturnTypes, 2),
5325 makeArrayRef(MTCTROps, Glue.getNode() ? 3 : 2));
5326 // The glue is the second value produced.
5327 Glue = Chain.getValue(1);
5328}
5329
5330static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5331 SDValue &Glue, SDValue &Chain,
5332 SDValue CallSeqStart,
5333 const CallBase *CB, const SDLoc &dl,
5334 bool hasNest,
5335 const PPCSubtarget &Subtarget) {
5336 // Function pointers in the 64-bit SVR4 ABI do not point to the function
5337 // entry point, but to the function descriptor (the function entry point
5338 // address is part of the function descriptor though).
5339 // The function descriptor is a three doubleword structure with the
5340 // following fields: function entry point, TOC base address and
5341 // environment pointer.
5342 // Thus for a call through a function pointer, the following actions need
5343 // to be performed:
5344 // 1. Save the TOC of the caller in the TOC save area of its stack
5345 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
5346 // 2. Load the address of the function entry point from the function
5347 // descriptor.
5348 // 3. Load the TOC of the callee from the function descriptor into r2.
5349 // 4. Load the environment pointer from the function descriptor into
5350 // r11.
5351 // 5. Branch to the function entry point address.
5352 // 6. On return of the callee, the TOC of the caller needs to be
5353 // restored (this is done in FinishCall()).
5354 //
5355 // The loads are scheduled at the beginning of the call sequence, and the
5356 // register copies are flagged together to ensure that no other
5357 // operations can be scheduled in between. E.g. without flagging the
5358 // copies together, a TOC access in the caller could be scheduled between
5359 // the assignment of the callee TOC and the branch to the callee, which leads
5360 // to incorrect code.
5361
5362 // Start by loading the function address from the descriptor.
5363 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart);
5364 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
5365 ? (MachineMemOperand::MODereferenceable |
5366 MachineMemOperand::MOInvariant)
5367 : MachineMemOperand::MONone;
5368
5369 MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr);
5370
5371 // Registers used in building the DAG.
5372 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister();
5373 const MCRegister TOCReg = Subtarget.getTOCPointerRegister();
5374
5375 // Offsets of descriptor members.
5376 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset();
5377 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset();
5378
5379 const MVT RegVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
5380 const unsigned Alignment = Subtarget.isPPC64() ? 8 : 4;
5381
5382 // One load for the functions entry point address.
5383 SDValue LoadFuncPtr = DAG.getLoad(RegVT, dl, LDChain, Callee, MPI,
5384 Alignment, MMOFlags);
5385
5386 // One for loading the TOC anchor for the module that contains the called
5387 // function.
5388 SDValue TOCOff = DAG.getIntPtrConstant(TOCAnchorOffset, dl);
5389 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, Callee, TOCOff);
5390 SDValue TOCPtr =
5391 DAG.getLoad(RegVT, dl, LDChain, AddTOC,
5392 MPI.getWithOffset(TOCAnchorOffset), Alignment, MMOFlags);
5393
5394 // One for loading the environment pointer.
5395 SDValue PtrOff = DAG.getIntPtrConstant(EnvPtrOffset, dl);
5396 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, RegVT, Callee, PtrOff);
5397 SDValue LoadEnvPtr =
5398 DAG.getLoad(RegVT, dl, LDChain, AddPtr,
5399 MPI.getWithOffset(EnvPtrOffset), Alignment, MMOFlags);
5400
5401
5402 // Then copy the newly loaded TOC anchor to the TOC pointer.
5403 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, TOCReg, TOCPtr, Glue);
5404 Chain = TOCVal.getValue(0);
5405 Glue = TOCVal.getValue(1);
5406
5407 // If the function call has an explicit 'nest' parameter, it takes the
5408 // place of the environment pointer.
5409 assert((!hasNest || !Subtarget.isAIXABI()) &&(static_cast <bool> ((!hasNest || !Subtarget.isAIXABI()
) && "Nest parameter is not supported on AIX.") ? void
(0) : __assert_fail ("(!hasNest || !Subtarget.isAIXABI()) && \"Nest parameter is not supported on AIX.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5410, __extension__ __PRETTY_FUNCTION__))
5410 "Nest parameter is not supported on AIX.")(static_cast <bool> ((!hasNest || !Subtarget.isAIXABI()
) && "Nest parameter is not supported on AIX.") ? void
(0) : __assert_fail ("(!hasNest || !Subtarget.isAIXABI()) && \"Nest parameter is not supported on AIX.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5410, __extension__ __PRETTY_FUNCTION__))
;
5411 if (!hasNest) {
5412 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, EnvPtrReg, LoadEnvPtr, Glue);
5413 Chain = EnvVal.getValue(0);
5414 Glue = EnvVal.getValue(1);
5415 }
5416
5417 // The rest of the indirect call sequence is the same as the non-descriptor
5418 // DAG.
5419 prepareIndirectCall(DAG, LoadFuncPtr, Glue, Chain, dl);
5420}
5421
5422static void
5423buildCallOperands(SmallVectorImpl<SDValue> &Ops,
5424 PPCTargetLowering::CallFlags CFlags, const SDLoc &dl,
5425 SelectionDAG &DAG,
5426 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
5427 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff,
5428 const PPCSubtarget &Subtarget) {
5429 const bool IsPPC64 = Subtarget.isPPC64();
5430 // MVT for a general purpose register.
5431 const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32;
5432
5433 // First operand is always the chain.
5434 Ops.push_back(Chain);
5435
5436 // If it's a direct call pass the callee as the second operand.
5437 if (!CFlags.IsIndirect)
5438 Ops.push_back(Callee);
5439 else {
5440 assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect.")(static_cast <bool> (!CFlags.IsPatchPoint && "Patch point calls are not indirect."
) ? void (0) : __assert_fail ("!CFlags.IsPatchPoint && \"Patch point calls are not indirect.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5440, __extension__ __PRETTY_FUNCTION__))
;
5441
5442 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area
5443 // on the stack (this would have been done in `LowerCall_64SVR4` or
5444 // `LowerCall_AIX`). The call instruction is a pseudo instruction that
5445 // represents both the indirect branch and a load that restores the TOC
5446 // pointer from the linkage area. The operand for the TOC restore is an add
5447 // of the TOC save offset to the stack pointer. This must be the second
5448 // operand: after the chain input but before any other variadic arguments.
5449 // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not
5450 // saved or used.
5451 if (isTOCSaveRestoreRequired(Subtarget)) {
5452 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
5453
5454 SDValue StackPtr = DAG.getRegister(StackPtrReg, RegVT);
5455 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5456 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
5457 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, StackPtr, TOCOff);
5458 Ops.push_back(AddTOC);
5459 }
5460
5461 // Add the register used for the environment pointer.
5462 if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest)
5463 Ops.push_back(DAG.getRegister(Subtarget.getEnvironmentPointerRegister(),
5464 RegVT));
5465
5466
5467 // Add CTR register as callee so a bctr can be emitted later.
5468 if (CFlags.IsTailCall)
5469 Ops.push_back(DAG.getRegister(IsPPC64 ? PPC::CTR8 : PPC::CTR, RegVT));
5470 }
5471
5472 // If this is a tail call add stack pointer delta.
5473 if (CFlags.IsTailCall)
5474 Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32));
5475
5476 // Add argument registers to the end of the list so that they are known live
5477 // into the call.
5478 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
5479 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
5480 RegsToPass[i].second.getValueType()));
5481
5482 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
5483 // no way to mark dependencies as implicit here.
5484 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter.
5485 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) &&
5486 !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls())
5487 Ops.push_back(DAG.getRegister(Subtarget.getTOCPointerRegister(), RegVT));
5488
5489 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
5490 if (CFlags.IsVarArg && Subtarget.is32BitELFABI())
5491 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32));
5492
5493 // Add a register mask operand representing the call-preserved registers.
5494 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
5495 const uint32_t *Mask =
5496 TRI->getCallPreservedMask(DAG.getMachineFunction(), CFlags.CallConv);
5497 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5497, __extension__ __PRETTY_FUNCTION__))
;
5498 Ops.push_back(DAG.getRegisterMask(Mask));
5499
5500 // If the glue is valid, it is the last operand.
5501 if (Glue.getNode())
5502 Ops.push_back(Glue);
5503}
5504
5505SDValue PPCTargetLowering::FinishCall(
5506 CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG,
5507 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue,
5508 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5509 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins,
5510 SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const {
5511
5512 if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) ||
5513 Subtarget.isAIXABI())
5514 setUsesTOCBasePtr(DAG);
5515
5516 unsigned CallOpc =
5517 getCallOpcode(CFlags, DAG.getMachineFunction().getFunction(), Callee,
5518 Subtarget, DAG.getTarget());
5519
5520 if (!CFlags.IsIndirect)
5521 Callee = transformCallee(Callee, DAG, dl, Subtarget);
5522 else if (Subtarget.usesFunctionDescriptors())
5523 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB,
5524 dl, CFlags.HasNest, Subtarget);
5525 else
5526 prepareIndirectCall(DAG, Callee, Glue, Chain, dl);
5527
5528 // Build the operand list for the call instruction.
5529 SmallVector<SDValue, 8> Ops;
5530 buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee,
5531 SPDiff, Subtarget);
5532
5533 // Emit tail call.
5534 if (CFlags.IsTailCall) {
5535 // Indirect tail call when using PC Relative calls do not have the same
5536 // constraints.
5537 assert(((Callee.getOpcode() == ISD::Register &&(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5538 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5539 Callee.getOpcode() == ISD::TargetExternalSymbol ||(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5540 Callee.getOpcode() == ISD::TargetGlobalAddress ||(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5541 isa<ConstantSDNode>(Callee) ||(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5542 (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5543 "Expecting a global address, external symbol, absolute value, "(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5544 "register or an indirect tail call when PC Relative calls are "(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
5545 "used.")(static_cast <bool> (((Callee.getOpcode() == ISD::Register
&& cast<RegisterSDNode>(Callee)->getReg() ==
PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol
|| Callee.getOpcode() == ISD::TargetGlobalAddress || isa<
ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget
.isUsingPCRelativeCalls())) && "Expecting a global address, external symbol, absolute value, "
"register or an indirect tail call when PC Relative calls are "
"used.") ? void (0) : __assert_fail ("((Callee.getOpcode() == ISD::Register && cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa<ConstantSDNode>(Callee) || (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) && \"Expecting a global address, external symbol, absolute value, \" \"register or an indirect tail call when PC Relative calls are \" \"used.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__))
;
5546 // PC Relative calls also use TC_RETURN as the way to mark tail calls.
5547 assert(CallOpc == PPCISD::TC_RETURN &&(static_cast <bool> (CallOpc == PPCISD::TC_RETURN &&
"Unexpected call opcode for a tail call.") ? void (0) : __assert_fail
("CallOpc == PPCISD::TC_RETURN && \"Unexpected call opcode for a tail call.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5548, __extension__ __PRETTY_FUNCTION__))
5548 "Unexpected call opcode for a tail call.")(static_cast <bool> (CallOpc == PPCISD::TC_RETURN &&
"Unexpected call opcode for a tail call.") ? void (0) : __assert_fail
("CallOpc == PPCISD::TC_RETURN && \"Unexpected call opcode for a tail call.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5548, __extension__ __PRETTY_FUNCTION__))
;
5549 DAG.getMachineFunction().getFrameInfo().setHasTailCall();
5550 return DAG.getNode(CallOpc, dl, MVT::Other, Ops);
5551 }
5552
5553 std::array<EVT, 2> ReturnTypes = {{MVT::Other, MVT::Glue}};
5554 Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops);
5555 DAG.addNoMergeSiteInfo(Chain.getNode(), CFlags.NoMerge);
5556 Glue = Chain.getValue(1);
5557
5558 // When performing tail call optimization the callee pops its arguments off
5559 // the stack. Account for this here so these bytes can be pushed back on in
5560 // PPCFrameLowering::eliminateCallFramePseudoInstr.
5561 int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast &&
5562 getTargetMachine().Options.GuaranteedTailCallOpt)
5563 ? NumBytes
5564 : 0;
5565
5566 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
5567 DAG.getIntPtrConstant(BytesCalleePops, dl, true),
5568 Glue, dl);
5569 Glue = Chain.getValue(1);
5570
5571 return LowerCallResult(Chain, Glue, CFlags.CallConv, CFlags.IsVarArg, Ins, dl,
5572 DAG, InVals);
5573}
5574
5575SDValue
5576PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
5577 SmallVectorImpl<SDValue> &InVals) const {
5578 SelectionDAG &DAG = CLI.DAG;
5579 SDLoc &dl = CLI.DL;
5580 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
5581 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
5582 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
5583 SDValue Chain = CLI.Chain;
5584 SDValue Callee = CLI.Callee;
5585 bool &isTailCall = CLI.IsTailCall;
5586 CallingConv::ID CallConv = CLI.CallConv;
5587 bool isVarArg = CLI.IsVarArg;
5588 bool isPatchPoint = CLI.IsPatchPoint;
5589 const CallBase *CB = CLI.CB;
5590
5591 if (isTailCall) {
5592 if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall()))
5593 isTailCall = false;
5594 else if (Subtarget.isSVR4ABI() && Subtarget.isPPC64())
5595 isTailCall = IsEligibleForTailCallOptimization_64SVR4(
5596 Callee, CallConv, CB, isVarArg, Outs, Ins, DAG);
5597 else
5598 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg,
5599 Ins, DAG);
5600 if (isTailCall) {
5601 ++NumTailCalls;
5602 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5603 ++NumSiblingCalls;
5604
5605 // PC Relative calls no longer guarantee that the callee is a Global
5606 // Address Node. The callee could be an indirect tail call in which
5607 // case the SDValue for the callee could be a load (to load the address
5608 // of a function pointer) or it may be a register copy (to move the
5609 // address of the callee from a function parameter into a virtual
5610 // register). It may also be an ExternalSymbolSDNode (ex memcopy).
5611 assert((Subtarget.isUsingPCRelativeCalls() ||(static_cast <bool> ((Subtarget.isUsingPCRelativeCalls(
) || isa<GlobalAddressSDNode>(Callee)) && "Callee should be an llvm::Function object."
) ? void (0) : __assert_fail ("(Subtarget.isUsingPCRelativeCalls() || isa<GlobalAddressSDNode>(Callee)) && \"Callee should be an llvm::Function object.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5613, __extension__ __PRETTY_FUNCTION__))
5612 isa<GlobalAddressSDNode>(Callee)) &&(static_cast <bool> ((Subtarget.isUsingPCRelativeCalls(
) || isa<GlobalAddressSDNode>(Callee)) && "Callee should be an llvm::Function object."
) ? void (0) : __assert_fail ("(Subtarget.isUsingPCRelativeCalls() || isa<GlobalAddressSDNode>(Callee)) && \"Callee should be an llvm::Function object.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5613, __extension__ __PRETTY_FUNCTION__))
5613 "Callee should be an llvm::Function object.")(static_cast <bool> ((Subtarget.isUsingPCRelativeCalls(
) || isa<GlobalAddressSDNode>(Callee)) && "Callee should be an llvm::Function object."
) ? void (0) : __assert_fail ("(Subtarget.isUsingPCRelativeCalls() || isa<GlobalAddressSDNode>(Callee)) && \"Callee should be an llvm::Function object.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5613, __extension__ __PRETTY_FUNCTION__))
;
5614
5615 LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { dbgs() << "TCO caller: " << DAG
.getMachineFunction().getName() << "\nTCO callee: "; } }
while (false)
5616 << "\nTCO callee: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { dbgs() << "TCO caller: " << DAG
.getMachineFunction().getName() << "\nTCO callee: "; } }
while (false)
;
5617 LLVM_DEBUG(Callee.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { Callee.dump(); } } while (false)
;
5618 }
5619 }
5620
5621 if (!isTailCall && CB && CB->isMustTailCall())
5622 report_fatal_error("failed to perform tail call elimination on a call "
5623 "site marked musttail");
5624
5625 // When long calls (i.e. indirect calls) are always used, calls are always
5626 // made via function pointer. If we have a function name, first translate it
5627 // into a pointer.
5628 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Callee) &&
5629 !isTailCall)
5630 Callee = LowerGlobalAddress(Callee, DAG);
5631
5632 CallFlags CFlags(
5633 CallConv, isTailCall, isVarArg, isPatchPoint,
5634 isIndirectCall(Callee, DAG, Subtarget, isPatchPoint),
5635 // hasNest
5636 Subtarget.is64BitELFABI() &&
5637 any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }),
5638 CLI.NoMerge);
5639
5640 if (Subtarget.isAIXABI())
5641 return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5642 InVals, CB);
5643
5644 assert(Subtarget.isSVR4ABI())(static_cast <bool> (Subtarget.isSVR4ABI()) ? void (0) :
__assert_fail ("Subtarget.isSVR4ABI()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5644, __extension__ __PRETTY_FUNCTION__))
;
5645 if (Subtarget.isPPC64())
5646 return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5647 InVals, CB);
5648 return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5649 InVals, CB);
5650}
5651
5652SDValue PPCTargetLowering::LowerCall_32SVR4(
5653 SDValue Chain, SDValue Callee, CallFlags CFlags,
5654 const SmallVectorImpl<ISD::OutputArg> &Outs,
5655 const SmallVectorImpl<SDValue> &OutVals,
5656 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5657 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5658 const CallBase *CB) const {
5659 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
5660 // of the 32-bit SVR4 ABI stack frame layout.
5661
5662 const CallingConv::ID CallConv = CFlags.CallConv;
5663 const bool IsVarArg = CFlags.IsVarArg;
5664 const bool IsTailCall = CFlags.IsTailCall;
5665
5666 assert((CallConv == CallingConv::C ||(static_cast <bool> ((CallConv == CallingConv::C || CallConv
== CallingConv::Cold || CallConv == CallingConv::Fast) &&
"Unknown calling convention!") ? void (0) : __assert_fail ("(CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && \"Unknown calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5668, __extension__ __PRETTY_FUNCTION__))
5667 CallConv == CallingConv::Cold ||(static_cast <bool> ((CallConv == CallingConv::C || CallConv
== CallingConv::Cold || CallConv == CallingConv::Fast) &&
"Unknown calling convention!") ? void (0) : __assert_fail ("(CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && \"Unknown calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5668, __extension__ __PRETTY_FUNCTION__))
5668 CallConv == CallingConv::Fast) && "Unknown calling convention!")(static_cast <bool> ((CallConv == CallingConv::C || CallConv
== CallingConv::Cold || CallConv == CallingConv::Fast) &&
"Unknown calling convention!") ? void (0) : __assert_fail ("(CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && \"Unknown calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5668, __extension__ __PRETTY_FUNCTION__))
;
5669
5670 const Align PtrAlign(4);
5671
5672 MachineFunction &MF = DAG.getMachineFunction();
5673
5674 // Mark this function as potentially containing a function that contains a
5675 // tail call. As a consequence the frame pointer will be used for dynamicalloc
5676 // and restoring the callers stack pointer in this functions epilog. This is
5677 // done because by tail calling the called function might overwrite the value
5678 // in this function's (MF) stack pointer stack slot 0(SP).
5679 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
5680 CallConv == CallingConv::Fast)
5681 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
5682
5683 // Count how many bytes are to be pushed on the stack, including the linkage
5684 // area, parameter list area and the part of the local variable space which
5685 // contains copies of aggregates which are passed by value.
5686
5687 // Assign locations to all of the outgoing arguments.
5688 SmallVector<CCValAssign, 16> ArgLocs;
5689 PPCCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
5690
5691 // Reserve space for the linkage area on the stack.
5692 CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(),
5693 PtrAlign);
5694 if (useSoftFloat())
5695 CCInfo.PreAnalyzeCallOperands(Outs);
5696
5697 if (IsVarArg) {
5698 // Handle fixed and variable vector arguments differently.
5699 // Fixed vector arguments go into registers as long as registers are
5700 // available. Variable vector arguments always go into memory.
5701 unsigned NumArgs = Outs.size();
5702
5703 for (unsigned i = 0; i != NumArgs; ++i) {
5704 MVT ArgVT = Outs[i].VT;
5705 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
5706 bool Result;
5707
5708 if (Outs[i].IsFixed) {
5709 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags,
5710 CCInfo);
5711 } else {
5712 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full,
5713 ArgFlags, CCInfo);
5714 }
5715
5716 if (Result) {
5717#ifndef NDEBUG
5718 errs() << "Call operand #" << i << " has unhandled type "
5719 << EVT(ArgVT).getEVTString() << "\n";
5720#endif
5721 llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5721)
;
5722 }
5723 }
5724 } else {
5725 // All arguments are treated the same.
5726 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4);
5727 }
5728 CCInfo.clearWasPPCF128();
5729
5730 // Assign locations to all of the outgoing aggregate by value arguments.
5731 SmallVector<CCValAssign, 16> ByValArgLocs;
5732 CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext());
5733
5734 // Reserve stack space for the allocations in CCInfo.
5735 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrAlign);
5736
5737 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal);
5738
5739 // Size of the linkage area, parameter list area and the part of the local
5740 // space variable where copies of aggregates which are passed by value are
5741 // stored.
5742 unsigned NumBytes = CCByValInfo.getNextStackOffset();
5743
5744 // Calculate by how many bytes the stack has to be adjusted in case of tail
5745 // call optimization.
5746 int SPDiff = CalculateTailCallSPDiff(DAG, IsTailCall, NumBytes);
5747
5748 // Adjust the stack pointer for the new arguments...
5749 // These operations are automatically eliminated by the prolog/epilog pass
5750 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
5751 SDValue CallSeqStart = Chain;
5752
5753 // Load the return address and frame pointer so it can be moved somewhere else
5754 // later.
5755 SDValue LROp, FPOp;
5756 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl);
5757
5758 // Set up a copy of the stack pointer for use loading and storing any
5759 // arguments that may not fit in the registers available for argument
5760 // passing.
5761 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
5762
5763 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
5764 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
5765 SmallVector<SDValue, 8> MemOpChains;
5766
5767 bool seenFloatArg = false;
5768 // Walk the register/memloc assignments, inserting copies/loads.
5769 // i - Tracks the index into the list of registers allocated for the call
5770 // RealArgIdx - Tracks the index into the list of actual function arguments
5771 // j - Tracks the index into the list of byval arguments
5772 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size();
5773 i != e;
5774 ++i, ++RealArgIdx) {
5775 CCValAssign &VA = ArgLocs[i];
5776 SDValue Arg = OutVals[RealArgIdx];
5777 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags;
5778
5779 if (Flags.isByVal()) {
5780 // Argument is an aggregate which is passed by value, thus we need to
5781 // create a copy of it in the local variable space of the current stack
5782 // frame (which is the stack frame of the caller) and pass the address of
5783 // this copy to the callee.
5784 assert((j < ByValArgLocs.size()) && "Index out of bounds!")(static_cast <bool> ((j < ByValArgLocs.size()) &&
"Index out of bounds!") ? void (0) : __assert_fail ("(j < ByValArgLocs.size()) && \"Index out of bounds!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5784, __extension__ __PRETTY_FUNCTION__))
;
5785 CCValAssign &ByValVA = ByValArgLocs[j++];
5786 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!")(static_cast <bool> ((VA.getValNo() == ByValVA.getValNo
()) && "ValNo mismatch!") ? void (0) : __assert_fail (
"(VA.getValNo() == ByValVA.getValNo()) && \"ValNo mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5786, __extension__ __PRETTY_FUNCTION__))
;
5787
5788 // Memory reserved in the local variable space of the callers stack frame.
5789 unsigned LocMemOffset = ByValVA.getLocMemOffset();
5790
5791 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
5792 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()),
5793 StackPtr, PtrOff);
5794
5795 // Create a copy of the argument in the local area of the current
5796 // stack frame.
5797 SDValue MemcpyCall =
5798 CreateCopyOfByValArgument(Arg, PtrOff,
5799 CallSeqStart.getNode()->getOperand(0),
5800 Flags, DAG, dl);
5801
5802 // This must go outside the CALLSEQ_START..END.
5803 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0,
5804 SDLoc(MemcpyCall));
5805 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
5806 NewCallSeqStart.getNode());
5807 Chain = CallSeqStart = NewCallSeqStart;
5808
5809 // Pass the address of the aggregate copy on the stack either in a
5810 // physical register or in the parameter list area of the current stack
5811 // frame to the callee.
5812 Arg = PtrOff;
5813 }
5814
5815 // When useCRBits() is true, there can be i1 arguments.
5816 // It is because getRegisterType(MVT::i1) => MVT::i1,
5817 // and for other integer types getRegisterType() => MVT::i32.
5818 // Extend i1 and ensure callee will get i32.
5819 if (Arg.getValueType() == MVT::i1)
5820 Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
5821 dl, MVT::i32, Arg);
5822
5823 if (VA.isRegLoc()) {
5824 seenFloatArg |= VA.getLocVT().isFloatingPoint();
5825 // Put argument in a physical register.
5826 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) {
5827 bool IsLE = Subtarget.isLittleEndian();
5828 SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
5829 DAG.getIntPtrConstant(IsLE ? 0 : 1, dl));
5830 RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0)));
5831 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
5832 DAG.getIntPtrConstant(IsLE ? 1 : 0, dl));
5833 RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(),
5834 SVal.getValue(0)));
5835 } else
5836 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
5837 } else {
5838 // Put argument in the parameter list area of the current stack frame.
5839 assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5839, __extension__ __PRETTY_FUNCTION__))
;
5840 unsigned LocMemOffset = VA.getLocMemOffset();
5841
5842 if (!IsTailCall) {
5843 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
5844 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()),
5845 StackPtr, PtrOff);
5846
5847 MemOpChains.push_back(
5848 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()));
5849 } else {
5850 // Calculate and remember argument location.
5851 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset,
5852 TailCallArguments);
5853 }
5854 }
5855 }
5856
5857 if (!MemOpChains.empty())
5858 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
5859
5860 // Build a sequence of copy-to-reg nodes chained together with token chain
5861 // and flag operands which copy the outgoing args into the appropriate regs.
5862 SDValue InFlag;
5863 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
5864 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
5865 RegsToPass[i].second, InFlag);
5866 InFlag = Chain.getValue(1);
5867 }
5868
5869 // Set CR bit 6 to true if this is a vararg call with floating args passed in
5870 // registers.
5871 if (IsVarArg) {
5872 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
5873 SDValue Ops[] = { Chain, InFlag };
5874
5875 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET,
5876 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1));
5877
5878 InFlag = Chain.getValue(1);
5879 }
5880
5881 if (IsTailCall)
5882 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
5883 TailCallArguments);
5884
5885 return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart,
5886 Callee, SPDiff, NumBytes, Ins, InVals, CB);
5887}
5888
5889// Copy an argument into memory, being careful to do this outside the
5890// call sequence for the call to which the argument belongs.
5891SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
5892 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
5893 SelectionDAG &DAG, const SDLoc &dl) const {
5894 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff,
5895 CallSeqStart.getNode()->getOperand(0),
5896 Flags, DAG, dl);
5897 // The MEMCPY must go outside the CALLSEQ_START..END.
5898 int64_t FrameSize = CallSeqStart.getConstantOperandVal(1);
5899 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0,
5900 SDLoc(MemcpyCall));
5901 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
5902 NewCallSeqStart.getNode());
5903 return NewCallSeqStart;
5904}
5905
5906SDValue PPCTargetLowering::LowerCall_64SVR4(
5907 SDValue Chain, SDValue Callee, CallFlags CFlags,
5908 const SmallVectorImpl<ISD::OutputArg> &Outs,
5909 const SmallVectorImpl<SDValue> &OutVals,
5910 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5911 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5912 const CallBase *CB) const {
5913 bool isELFv2ABI = Subtarget.isELFv2ABI();
5914 bool isLittleEndian = Subtarget.isLittleEndian();
5915 unsigned NumOps = Outs.size();
5916 bool IsSibCall = false;
5917 bool IsFastCall = CFlags.CallConv == CallingConv::Fast;
5918
5919 EVT PtrVT = getPointerTy(DAG.getDataLayout());
5920 unsigned PtrByteSize = 8;
5921
5922 MachineFunction &MF = DAG.getMachineFunction();
5923
5924 if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt)
5925 IsSibCall = true;
5926
5927 // Mark this function as potentially containing a function that contains a
5928 // tail call. As a consequence the frame pointer will be used for dynamicalloc
5929 // and restoring the callers stack pointer in this functions epilog. This is
5930 // done because by tail calling the called function might overwrite the value
5931 // in this function's (MF) stack pointer stack slot 0(SP).
5932 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
5933 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
5934
5935 assert(!(IsFastCall && CFlags.IsVarArg) &&(static_cast <bool> (!(IsFastCall && CFlags.IsVarArg
) && "fastcc not supported on varargs functions") ? void
(0) : __assert_fail ("!(IsFastCall && CFlags.IsVarArg) && \"fastcc not supported on varargs functions\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5936, __extension__ __PRETTY_FUNCTION__))
5936 "fastcc not supported on varargs functions")(static_cast <bool> (!(IsFastCall && CFlags.IsVarArg
) && "fastcc not supported on varargs functions") ? void
(0) : __assert_fail ("!(IsFastCall && CFlags.IsVarArg) && \"fastcc not supported on varargs functions\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 5936, __extension__ __PRETTY_FUNCTION__))
;
5937
5938 // Count how many bytes are to be pushed on the stack, including the linkage
5939 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes
5940 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
5941 // area is 32 bytes reserved space for [SP][CR][LR][TOC].
5942 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
5943 unsigned NumBytes = LinkageSize;
5944 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
5945
5946 static const MCPhysReg GPR[] = {
5947 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
5948 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
5949 };
5950 static const MCPhysReg VR[] = {
5951 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
5952 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
5953 };
5954
5955 const unsigned NumGPRs = array_lengthof(GPR);
5956 const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
5957 const unsigned NumVRs = array_lengthof(VR);
5958
5959 // On ELFv2, we can avoid allocating the parameter area if all the arguments
5960 // can be passed to the callee in registers.
5961 // For the fast calling convention, there is another check below.
5962 // Note: We should keep consistent with LowerFormalArguments_64SVR4()
5963 bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall;
5964 if (!HasParameterArea) {
5965 unsigned ParamAreaSize = NumGPRs * PtrByteSize;
5966 unsigned AvailableFPRs = NumFPRs;
5967 unsigned AvailableVRs = NumVRs;
5968 unsigned NumBytesTmp = NumBytes;
5969 for (unsigned i = 0; i != NumOps; ++i) {
5970 if (Outs[i].Flags.isNest()) continue;
5971 if (CalculateStackSlotUsed(Subtarget, Outs[i].VT, Outs[i].ArgVT,
5972 Outs[i].Flags, PtrByteSize, LinkageSize,
5973 ParamAreaSize, NumBytesTmp, AvailableFPRs,
5974 AvailableVRs))
5975 HasParameterArea = true;
5976 }
5977 }
5978
5979 // When using the fast calling convention, we don't provide backing for
5980 // arguments that will be in registers.
5981 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
5982
5983 // Avoid allocating parameter area for fastcc functions if all the arguments
5984 // can be passed in the registers.
5985 if (IsFastCall)
5986 HasParameterArea = false;
5987
5988 // Add up all the space actually used.
5989 for (unsigned i = 0; i != NumOps; ++i) {
5990 ISD::ArgFlagsTy Flags = Outs[i].Flags;
5991 EVT ArgVT = Outs[i].VT;
5992 EVT OrigVT = Outs[i].ArgVT;
5993
5994 if (Flags.isNest())
5995 continue;
5996
5997 if (IsFastCall) {
5998 if (Flags.isByVal()) {
5999 NumGPRsUsed += (Flags.getByValSize()+7)/8;
6000 if (NumGPRsUsed > NumGPRs)
6001 HasParameterArea = true;
6002 } else {
6003 switch (ArgVT.getSimpleVT().SimpleTy) {
6004 default: llvm_unreachable("Unexpected ValueType for argument!")::llvm::llvm_unreachable_internal("Unexpected ValueType for argument!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6004)
;
6005 case MVT::i1:
6006 case MVT::i32:
6007 case MVT::i64:
6008 if (++NumGPRsUsed <= NumGPRs)
6009 continue;
6010 break;
6011 case MVT::v4i32:
6012 case MVT::v8i16:
6013 case MVT::v16i8:
6014 case MVT::v2f64:
6015 case MVT::v2i64:
6016 case MVT::v1i128:
6017 case MVT::f128:
6018 if (++NumVRsUsed <= NumVRs)
6019 continue;
6020 break;
6021 case MVT::v4f32:
6022 if (++NumVRsUsed <= NumVRs)
6023 continue;
6024 break;
6025 case MVT::f32:
6026 case MVT::f64:
6027 if (++NumFPRsUsed <= NumFPRs)
6028 continue;
6029 break;
6030 }
6031 HasParameterArea = true;
6032 }
6033 }
6034
6035 /* Respect alignment of argument on the stack. */
6036 auto Alignement =
6037 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6038 NumBytes = alignTo(NumBytes, Alignement);
6039
6040 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
6041 if (Flags.isInConsecutiveRegsLast())
6042 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6043 }
6044
6045 unsigned NumBytesActuallyUsed = NumBytes;
6046
6047 // In the old ELFv1 ABI,
6048 // the prolog code of the callee may store up to 8 GPR argument registers to
6049 // the stack, allowing va_start to index over them in memory if its varargs.
6050 // Because we cannot tell if this is needed on the caller side, we have to
6051 // conservatively assume that it is needed. As such, make sure we have at
6052 // least enough stack space for the caller to store the 8 GPRs.
6053 // In the ELFv2 ABI, we allocate the parameter area iff a callee
6054 // really requires memory operands, e.g. a vararg function.
6055 if (HasParameterArea)
6056 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize);
6057 else
6058 NumBytes = LinkageSize;
6059
6060 // Tail call needs the stack to be aligned.
6061 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6062 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes);
6063
6064 int SPDiff = 0;
6065
6066 // Calculate by how many bytes the stack has to be adjusted in case of tail
6067 // call optimization.
6068 if (!IsSibCall)
6069 SPDiff = CalculateTailCallSPDiff(DAG, CFlags.IsTailCall, NumBytes);
6070
6071 // To protect arguments on the stack from being clobbered in a tail call,
6072 // force all the loads to happen before doing any other lowering.
6073 if (CFlags.IsTailCall)
6074 Chain = DAG.getStackArgumentTokenFactor(Chain);
6075
6076 // Adjust the stack pointer for the new arguments...
6077 // These operations are automatically eliminated by the prolog/epilog pass
6078 if (!IsSibCall)
6079 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
6080 SDValue CallSeqStart = Chain;
6081
6082 // Load the return address and frame pointer so it can be move somewhere else
6083 // later.
6084 SDValue LROp, FPOp;
6085 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl);
6086
6087 // Set up a copy of the stack pointer for use loading and storing any
6088 // arguments that may not fit in the registers available for argument
6089 // passing.
6090 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
6091
6092 // Figure out which arguments are going to go in registers, and which in
6093 // memory. Also, if this is a vararg function, floating point operations
6094 // must be stored to our stack, and loaded into integer regs as well, if
6095 // any integer regs are available for argument passing.
6096 unsigned ArgOffset = LinkageSize;
6097
6098 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6099 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6100
6101 SmallVector<SDValue, 8> MemOpChains;
6102 for (unsigned i = 0; i != NumOps; ++i) {
6103 SDValue Arg = OutVals[i];
6104 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6105 EVT ArgVT = Outs[i].VT;
6106 EVT OrigVT = Outs[i].ArgVT;
6107
6108 // PtrOff will be used to store the current argument to the stack if a
6109 // register cannot be found for it.
6110 SDValue PtrOff;
6111
6112 // We re-align the argument offset for each argument, except when using the
6113 // fast calling convention, when we need to make sure we do that only when
6114 // we'll actually use a stack slot.
6115 auto ComputePtrOff = [&]() {
6116 /* Respect alignment of argument on the stack. */
6117 auto Alignment =
6118 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6119 ArgOffset = alignTo(ArgOffset, Alignment);
6120
6121 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType());
6122
6123 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
6124 };
6125
6126 if (!IsFastCall) {
6127 ComputePtrOff();
6128
6129 /* Compute GPR index associated with argument offset. */
6130 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
6131 GPR_idx = std::min(GPR_idx, NumGPRs);
6132 }
6133
6134 // Promote integers to 64-bit values.
6135 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
6136 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
6137 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6138 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg);
6139 }
6140
6141 // FIXME memcpy is used way more than necessary. Correctness first.
6142 // Note: "by value" is code for passing a structure by value, not
6143 // basic types.
6144 if (Flags.isByVal()) {
6145 // Note: Size includes alignment padding, so
6146 // struct x { short a; char b; }
6147 // will have Size = 4. With #pragma pack(1), it will have Size = 3.
6148 // These are the proper values we need for right-justifying the
6149 // aggregate in a parameter register.
6150 unsigned Size = Flags.getByValSize();
6151
6152 // An empty aggregate parameter takes up no storage and no
6153 // registers.
6154 if (Size == 0)
6155 continue;
6156
6157 if (IsFastCall)
6158 ComputePtrOff();
6159
6160 // All aggregates smaller than 8 bytes must be passed right-justified.
6161 if (Size==1 || Size==2 || Size==4) {
6162 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
6163 if (GPR_idx != NumGPRs) {
6164 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
6165 MachinePointerInfo(), VT);
6166 MemOpChains.push_back(Load.getValue(1));
6167 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
6168
6169 ArgOffset += PtrByteSize;
6170 continue;
6171 }
6172 }
6173
6174 if (GPR_idx == NumGPRs && Size < 8) {
6175 SDValue AddPtr = PtrOff;
6176 if (!isLittleEndian) {
6177 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
6178 PtrOff.getValueType());
6179 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
6180 }
6181 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
6182 CallSeqStart,
6183 Flags, DAG, dl);
6184 ArgOffset += PtrByteSize;
6185 continue;
6186 }
6187 // Copy entire object into memory. There are cases where gcc-generated
6188 // code assumes it is there, even if it could be put entirely into
6189 // registers. (This is not what the doc says.)
6190
6191 // FIXME: The above statement is likely due to a misunderstanding of the
6192 // documents. All arguments must be copied into the parameter area BY
6193 // THE CALLEE in the event that the callee takes the address of any
6194 // formal argument. That has not yet been implemented. However, it is
6195 // reasonable to use the stack area as a staging area for the register
6196 // load.
6197
6198 // Skip this for small aggregates, as we will use the same slot for a
6199 // right-justified copy, below.
6200 if (Size >= 8)
6201 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
6202 CallSeqStart,
6203 Flags, DAG, dl);
6204
6205 // When a register is available, pass a small aggregate right-justified.
6206 if (Size < 8 && GPR_idx != NumGPRs) {
6207 // The easiest way to get this right-justified in a register
6208 // is to copy the structure into the rightmost portion of a
6209 // local variable slot, then load the whole slot into the
6210 // register.
6211 // FIXME: The memcpy seems to produce pretty awful code for
6212 // small aggregates, particularly for packed ones.
6213 // FIXME: It would be preferable to use the slot in the
6214 // parameter save area instead of a new local variable.
6215 SDValue AddPtr = PtrOff;
6216 if (!isLittleEndian) {
6217 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType());
6218 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
6219 }
6220 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
6221 CallSeqStart,
6222 Flags, DAG, dl);
6223
6224 // Load the slot into the register.
6225 SDValue Load =
6226 DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo());
6227 MemOpChains.push_back(Load.getValue(1));
6228 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
6229
6230 // Done with this argument.
6231 ArgOffset += PtrByteSize;
6232 continue;
6233 }
6234
6235 // For aggregates larger than PtrByteSize, copy the pieces of the
6236 // object that fit into registers from the parameter save area.
6237 for (unsigned j=0; j<Size; j+=PtrByteSize) {
6238 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
6239 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
6240 if (GPR_idx != NumGPRs) {
6241 SDValue Load =
6242 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo());
6243 MemOpChains.push_back(Load.getValue(1));
6244 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
6245 ArgOffset += PtrByteSize;
6246 } else {
6247 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
6248 break;
6249 }
6250 }
6251 continue;
6252 }
6253
6254 switch (Arg.getSimpleValueType().SimpleTy) {
6255 default: llvm_unreachable("Unexpected ValueType for argument!")::llvm::llvm_unreachable_internal("Unexpected ValueType for argument!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6255)
;
6256 case MVT::i1:
6257 case MVT::i32:
6258 case MVT::i64:
6259 if (Flags.isNest()) {
6260 // The 'nest' parameter, if any, is passed in R11.
6261 RegsToPass.push_back(std::make_pair(PPC::X11, Arg));
6262 break;
6263 }
6264
6265 // These can be scalar arguments or elements of an integer array type
6266 // passed directly. Clang may use those instead of "byval" aggregate
6267 // types to avoid forcing arguments to memory unnecessarily.
6268 if (GPR_idx != NumGPRs) {
6269 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg));
6270 } else {
6271 if (IsFastCall)
6272 ComputePtrOff();
6273
6274 assert(HasParameterArea &&(static_cast <bool> (HasParameterArea && "Parameter area must exist to pass an argument in memory."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist to pass an argument in memory.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6275, __extension__ __PRETTY_FUNCTION__))
6275 "Parameter area must exist to pass an argument in memory.")(static_cast <bool> (HasParameterArea && "Parameter area must exist to pass an argument in memory."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist to pass an argument in memory.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6275, __extension__ __PRETTY_FUNCTION__))
;
6276 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6277 true, CFlags.IsTailCall, false, MemOpChains,
6278 TailCallArguments, dl);
6279 if (IsFastCall)
6280 ArgOffset += PtrByteSize;
6281 }
6282 if (!IsFastCall)
6283 ArgOffset += PtrByteSize;
6284 break;
6285 case MVT::f32:
6286 case MVT::f64: {
6287 // These can be scalar arguments or elements of a float array type
6288 // passed directly. The latter are used to implement ELFv2 homogenous
6289 // float aggregates.
6290
6291 // Named arguments go into FPRs first, and once they overflow, the
6292 // remaining arguments go into GPRs and then the parameter save area.
6293 // Unnamed arguments for vararg functions always go to GPRs and
6294 // then the parameter save area. For now, put all arguments to vararg
6295 // routines always in both locations (FPR *and* GPR or stack slot).
6296 bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs;
6297 bool NeededLoad = false;
6298
6299 // First load the argument into the next available FPR.
6300 if (FPR_idx != NumFPRs)
6301 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg));
6302
6303 // Next, load the argument into GPR or stack slot if needed.
6304 if (!NeedGPROrStack)
6305 ;
6306 else if (GPR_idx != NumGPRs && !IsFastCall) {
6307 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
6308 // once we support fp <-> gpr moves.
6309
6310 // In the non-vararg case, this can only ever happen in the
6311 // presence of f32 array types, since otherwise we never run
6312 // out of FPRs before running out of GPRs.
6313 SDValue ArgVal;
6314
6315 // Double values are always passed in a single GPR.
6316 if (Arg.getValueType() != MVT::f32) {
6317 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg);
6318
6319 // Non-array float values are extended and passed in a GPR.
6320 } else if (!Flags.isInConsecutiveRegs()) {
6321 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
6322 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
6323
6324 // If we have an array of floats, we collect every odd element
6325 // together with its predecessor into one GPR.
6326 } else if (ArgOffset % PtrByteSize != 0) {
6327 SDValue Lo, Hi;
6328 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]);
6329 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
6330 if (!isLittleEndian)
6331 std::swap(Lo, Hi);
6332 ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6333
6334 // The final element, if even, goes into the first half of a GPR.
6335 } else if (Flags.isInConsecutiveRegsLast()) {
6336 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
6337 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
6338 if (!isLittleEndian)
6339 ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal,
6340 DAG.getConstant(32, dl, MVT::i32));
6341
6342 // Non-final even elements are skipped; they will be handled
6343 // together the with subsequent argument on the next go-around.
6344 } else
6345 ArgVal = SDValue();
6346
6347 if (ArgVal.getNode())
6348 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal));
6349 } else {
6350 if (IsFastCall)
6351 ComputePtrOff();
6352
6353 // Single-precision floating-point values are mapped to the
6354 // second (rightmost) word of the stack doubleword.
6355 if (Arg.getValueType() == MVT::f32 &&
6356 !isLittleEndian && !Flags.isInConsecutiveRegs()) {
6357 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
6358 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
6359 }
6360
6361 assert(HasParameterArea &&(static_cast <bool> (HasParameterArea && "Parameter area must exist to pass an argument in memory."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist to pass an argument in memory.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6362, __extension__ __PRETTY_FUNCTION__))
6362 "Parameter area must exist to pass an argument in memory.")(static_cast <bool> (HasParameterArea && "Parameter area must exist to pass an argument in memory."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist to pass an argument in memory.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6362, __extension__ __PRETTY_FUNCTION__))
;
6363 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6364 true, CFlags.IsTailCall, false, MemOpChains,
6365 TailCallArguments, dl);
6366
6367 NeededLoad = true;
6368 }
6369 // When passing an array of floats, the array occupies consecutive
6370 // space in the argument area; only round up to the next doubleword
6371 // at the end of the array. Otherwise, each float takes 8 bytes.
6372 if (!IsFastCall || NeededLoad) {
6373 ArgOffset += (Arg.getValueType() == MVT::f32 &&
6374 Flags.isInConsecutiveRegs()) ? 4 : 8;
6375 if (Flags.isInConsecutiveRegsLast())
6376 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6377 }
6378 break;
6379 }
6380 case MVT::v4f32:
6381 case MVT::v4i32:
6382 case MVT::v8i16:
6383 case MVT::v16i8:
6384 case MVT::v2f64:
6385 case MVT::v2i64:
6386 case MVT::v1i128:
6387 case MVT::f128:
6388 // These can be scalar arguments or elements of a vector array type
6389 // passed directly. The latter are used to implement ELFv2 homogenous
6390 // vector aggregates.
6391
6392 // For a varargs call, named arguments go into VRs or on the stack as
6393 // usual; unnamed arguments always go to the stack or the corresponding
6394 // GPRs when within range. For now, we always put the value in both
6395 // locations (or even all three).
6396 if (CFlags.IsVarArg) {
6397 assert(HasParameterArea &&(static_cast <bool> (HasParameterArea && "Parameter area must exist if we have a varargs call."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist if we have a varargs call.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6398, __extension__ __PRETTY_FUNCTION__))
6398 "Parameter area must exist if we have a varargs call.")(static_cast <bool> (HasParameterArea && "Parameter area must exist if we have a varargs call."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist if we have a varargs call.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6398, __extension__ __PRETTY_FUNCTION__))
;
6399 // We could elide this store in the case where the object fits
6400 // entirely in R registers. Maybe later.
6401 SDValue Store =
6402 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
6403 MemOpChains.push_back(Store);
6404 if (VR_idx != NumVRs) {
6405 SDValue Load =
6406 DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo());
6407 MemOpChains.push_back(Load.getValue(1));
6408 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load));
6409 }
6410 ArgOffset += 16;
6411 for (unsigned i=0; i<16; i+=PtrByteSize) {
6412 if (GPR_idx == NumGPRs)
6413 break;
6414 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
6415 DAG.getConstant(i, dl, PtrVT));
6416 SDValue Load =
6417 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo());
6418 MemOpChains.push_back(Load.getValue(1));
6419 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
6420 }
6421 break;
6422 }
6423
6424 // Non-varargs Altivec params go into VRs or on the stack.
6425 if (VR_idx != NumVRs) {
6426 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg));
6427 } else {
6428 if (IsFastCall)
6429 ComputePtrOff();
6430
6431 assert(HasParameterArea &&(static_cast <bool> (HasParameterArea && "Parameter area must exist to pass an argument in memory."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist to pass an argument in memory.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6432, __extension__ __PRETTY_FUNCTION__))
6432 "Parameter area must exist to pass an argument in memory.")(static_cast <bool> (HasParameterArea && "Parameter area must exist to pass an argument in memory."
) ? void (0) : __assert_fail ("HasParameterArea && \"Parameter area must exist to pass an argument in memory.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6432, __extension__ __PRETTY_FUNCTION__))
;
6433 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6434 true, CFlags.IsTailCall, true, MemOpChains,
6435 TailCallArguments, dl);
6436 if (IsFastCall)
6437 ArgOffset += 16;
6438 }
6439
6440 if (!IsFastCall)
6441 ArgOffset += 16;
6442 break;
6443 }
6444 }
6445
6446 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) &&(static_cast <bool> ((!HasParameterArea || NumBytesActuallyUsed
== ArgOffset) && "mismatch in size of parameter area"
) ? void (0) : __assert_fail ("(!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && \"mismatch in size of parameter area\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6447, __extension__ __PRETTY_FUNCTION__))
6447 "mismatch in size of parameter area")(static_cast <bool> ((!HasParameterArea || NumBytesActuallyUsed
== ArgOffset) && "mismatch in size of parameter area"
) ? void (0) : __assert_fail ("(!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && \"mismatch in size of parameter area\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6447, __extension__ __PRETTY_FUNCTION__))
;
6448 (void)NumBytesActuallyUsed;
6449
6450 if (!MemOpChains.empty())
6451 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
6452
6453 // Check if this is an indirect call (MTCTR/BCTRL).
6454 // See prepareDescriptorIndirectCall and buildCallOperands for more
6455 // information about calls through function pointers in the 64-bit SVR4 ABI.
6456 if (CFlags.IsIndirect) {
6457 // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the
6458 // caller in the TOC save area.
6459 if (isTOCSaveRestoreRequired(Subtarget)) {
6460 assert(!CFlags.IsTailCall && "Indirect tails calls not supported")(static_cast <bool> (!CFlags.IsTailCall && "Indirect tails calls not supported"
) ? void (0) : __assert_fail ("!CFlags.IsTailCall && \"Indirect tails calls not supported\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6460, __extension__ __PRETTY_FUNCTION__))
;
6461 // Load r2 into a virtual register and store it to the TOC save area.
6462 setUsesTOCBasePtr(DAG);
6463 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
6464 // TOC save area offset.
6465 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
6466 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
6467 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
6468 Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr,
6469 MachinePointerInfo::getStack(
6470 DAG.getMachineFunction(), TOCSaveOffset));
6471 }
6472 // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
6473 // This does not mean the MTCTR instruction must use R12; it's easier
6474 // to model this as an extra parameter, so do that.
6475 if (isELFv2ABI && !CFlags.IsPatchPoint)
6476 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee));
6477 }
6478
6479 // Build a sequence of copy-to-reg nodes chained together with token chain
6480 // and flag operands which copy the outgoing args into the appropriate regs.
6481 SDValue InFlag;
6482 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
6483 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
6484 RegsToPass[i].second, InFlag);
6485 InFlag = Chain.getValue(1);
6486 }
6487
6488 if (CFlags.IsTailCall && !IsSibCall)
6489 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6490 TailCallArguments);
6491
6492 return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart,
6493 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6494}
6495
6496// Returns true when the shadow of a general purpose argument register
6497// in the parameter save area is aligned to at least 'RequiredAlign'.
6498static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
6499 assert(RequiredAlign.value() <= 16 &&(static_cast <bool> (RequiredAlign.value() <= 16 &&
"Required alignment greater than stack alignment.") ? void (
0) : __assert_fail ("RequiredAlign.value() <= 16 && \"Required alignment greater than stack alignment.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6500, __extension__ __PRETTY_FUNCTION__))
6500 "Required alignment greater than stack alignment.")(static_cast <bool> (RequiredAlign.value() <= 16 &&
"Required alignment greater than stack alignment.") ? void (
0) : __assert_fail ("RequiredAlign.value() <= 16 && \"Required alignment greater than stack alignment.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6500, __extension__ __PRETTY_FUNCTION__))
;
6501 switch (Reg) {
6502 default:
6503 report_fatal_error("called on invalid register.");
6504 case PPC::R5:
6505 case PPC::R9:
6506 case PPC::X3:
6507 case PPC::X5:
6508 case PPC::X7:
6509 case PPC::X9:
6510 // These registers are 16 byte aligned which is the most strict aligment
6511 // we can support.
6512 return true;
6513 case PPC::R3:
6514 case PPC::R7:
6515 case PPC::X4:
6516 case PPC::X6:
6517 case PPC::X8:
6518 case PPC::X10:
6519 // The shadow of these registers in the PSA is 8 byte aligned.
6520 return RequiredAlign <= 8;
6521 case PPC::R4:
6522 case PPC::R6:
6523 case PPC::R8:
6524 case PPC::R10:
6525 return RequiredAlign <= 4;
6526 }
6527}
6528
6529static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
6530 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
6531 CCState &S) {
6532 AIXCCState &State = static_cast<AIXCCState &>(S);
6533 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
6534 State.getMachineFunction().getSubtarget());
6535 const bool IsPPC64 = Subtarget.isPPC64();
6536 const Align PtrAlign = IsPPC64 ? Align(8) : Align(4);
6537 const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32;
6538
6539 if (ValVT == MVT::f128)
6540 report_fatal_error("f128 is unimplemented on AIX.");
6541
6542 if (ArgFlags.isNest())
6543 report_fatal_error("Nest arguments are unimplemented.");
6544
6545 static const MCPhysReg GPR_32[] = {// 32-bit registers.
6546 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
6547 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
6548 static const MCPhysReg GPR_64[] = {// 64-bit registers.
6549 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6550 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
6551
6552 static const MCPhysReg VR[] = {// Vector registers.
6553 PPC::V2, PPC::V3, PPC::V4, PPC::V5,
6554 PPC::V6, PPC::V7, PPC::V8, PPC::V9,
6555 PPC::V10, PPC::V11, PPC::V12, PPC::V13};
6556
6557 if (ArgFlags.isByVal()) {
6558 if (ArgFlags.getNonZeroByValAlign() > PtrAlign)
6559 report_fatal_error("Pass-by-value arguments with alignment greater than "
6560 "register width are not supported.");
6561
6562 const unsigned ByValSize = ArgFlags.getByValSize();
6563
6564 // An empty aggregate parameter takes up no storage and no registers,
6565 // but needs a MemLoc for a stack slot for the formal arguments side.
6566 if (ByValSize == 0) {
6567 State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE,
6568 State.getNextStackOffset(), RegVT,
6569 LocInfo));
6570 return false;
6571 }
6572
6573 const unsigned StackSize = alignTo(ByValSize, PtrAlign);
6574 unsigned Offset = State.AllocateStack(StackSize, PtrAlign);
6575 for (const unsigned E = Offset + StackSize; Offset < E;
6576 Offset += PtrAlign.value()) {
6577 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32))
6578 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo));
6579 else {
6580 State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE,
6581 Offset, MVT::INVALID_SIMPLE_VALUE_TYPE,
6582 LocInfo));
6583 break;
6584 }
6585 }
6586 return false;
6587 }
6588
6589 // Arguments always reserve parameter save area.
6590 switch (ValVT.SimpleTy) {
6591 default:
6592 report_fatal_error("Unhandled value type for argument.");
6593 case MVT::i64:
6594 // i64 arguments should have been split to i32 for PPC32.
6595 assert(IsPPC64 && "PPC32 should have split i64 values.")(static_cast <bool> (IsPPC64 && "PPC32 should have split i64 values."
) ? void (0) : __assert_fail ("IsPPC64 && \"PPC32 should have split i64 values.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6595, __extension__ __PRETTY_FUNCTION__))
;
6596 LLVM_FALLTHROUGH[[gnu::fallthrough]];
6597 case MVT::i1:
6598 case MVT::i32: {
6599 const unsigned Offset = State.AllocateStack(PtrAlign.value(), PtrAlign);
6600 // AIX integer arguments are always passed in register width.
6601 if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits())
6602 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt
6603 : CCValAssign::LocInfo::ZExt;
6604 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32))
6605 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo));
6606 else
6607 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, RegVT, LocInfo));
6608
6609 return false;
6610 }
6611 case MVT::f32:
6612 case MVT::f64: {
6613 // Parameter save area (PSA) is reserved even if the float passes in fpr.
6614 const unsigned StoreSize = LocVT.getStoreSize();
6615 // Floats are always 4-byte aligned in the PSA on AIX.
6616 // This includes f64 in 64-bit mode for ABI compatibility.
6617 const unsigned Offset =
6618 State.AllocateStack(IsPPC64 ? 8 : StoreSize, Align(4));
6619 unsigned FReg = State.AllocateReg(FPR);
6620 if (FReg)
6621 State.addLoc(CCValAssign::getReg(ValNo, ValVT, FReg, LocVT, LocInfo));
6622
6623 // Reserve and initialize GPRs or initialize the PSA as required.
6624 for (unsigned I = 0; I < StoreSize; I += PtrAlign.value()) {
6625 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) {
6626 assert(FReg && "An FPR should be available when a GPR is reserved.")(static_cast <bool> (FReg && "An FPR should be available when a GPR is reserved."
) ? void (0) : __assert_fail ("FReg && \"An FPR should be available when a GPR is reserved.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6626, __extension__ __PRETTY_FUNCTION__))
;
6627 if (State.isVarArg()) {
6628 // Successfully reserved GPRs are only initialized for vararg calls.
6629 // Custom handling is required for:
6630 // f64 in PPC32 needs to be split into 2 GPRs.
6631 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR.
6632 State.addLoc(
6633 CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo));
6634 }
6635 } else {
6636 // If there are insufficient GPRs, the PSA needs to be initialized.
6637 // Initialization occurs even if an FPR was initialized for
6638 // compatibility with the AIX XL compiler. The full memory for the
6639 // argument will be initialized even if a prior word is saved in GPR.
6640 // A custom memLoc is used when the argument also passes in FPR so
6641 // that the callee handling can skip over it easily.
6642 State.addLoc(
6643 FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT,
6644 LocInfo)
6645 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
6646 break;
6647 }
6648 }
6649
6650 return false;
6651 }
6652 case MVT::v4f32:
6653 case MVT::v4i32:
6654 case MVT::v8i16:
6655 case MVT::v16i8:
6656 case MVT::v2i64:
6657 case MVT::v2f64:
6658 case MVT::v1i128: {
6659 const unsigned VecSize = 16;
6660 const Align VecAlign(VecSize);
6661
6662 if (!State.isVarArg()) {
6663 // If there are vector registers remaining we don't consume any stack
6664 // space.
6665 if (unsigned VReg = State.AllocateReg(VR)) {
6666 State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo));
6667 return false;
6668 }
6669 // Vectors passed on the stack do not shadow GPRs or FPRs even though they
6670 // might be allocated in the portion of the PSA that is shadowed by the
6671 // GPRs.
6672 const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
6673 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
6674 return false;
6675 }
6676
6677 const unsigned PtrSize = IsPPC64 ? 8 : 4;
6678 ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
6679
6680 unsigned NextRegIndex = State.getFirstUnallocated(GPRs);
6681 // Burn any underaligned registers and their shadowed stack space until
6682 // we reach the required alignment.
6683 while (NextRegIndex != GPRs.size() &&
6684 !isGPRShadowAligned(GPRs[NextRegIndex], VecAlign)) {
6685 // Shadow allocate register and its stack shadow.
6686 unsigned Reg = State.AllocateReg(GPRs);
6687 State.AllocateStack(PtrSize, PtrAlign);
6688 assert(Reg && "Allocating register unexpectedly failed.")(static_cast <bool> (Reg && "Allocating register unexpectedly failed."
) ? void (0) : __assert_fail ("Reg && \"Allocating register unexpectedly failed.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6688, __extension__ __PRETTY_FUNCTION__))
;
6689 (void)Reg;
6690 NextRegIndex = State.getFirstUnallocated(GPRs);
6691 }
6692
6693 // Vectors that are passed as fixed arguments are handled differently.
6694 // They are passed in VRs if any are available (unlike arguments passed
6695 // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
6696 // functions)
6697 if (State.isFixed(ValNo)) {
6698 if (unsigned VReg = State.AllocateReg(VR)) {
6699 State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo));
6700 // Shadow allocate GPRs and stack space even though we pass in a VR.
6701 for (unsigned I = 0; I != VecSize; I += PtrSize)
6702 State.AllocateReg(GPRs);
6703 State.AllocateStack(VecSize, VecAlign);
6704 return false;
6705 }
6706 // No vector registers remain so pass on the stack.
6707 const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
6708 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
6709 return false;
6710 }
6711
6712 // If all GPRS are consumed then we pass the argument fully on the stack.
6713 if (NextRegIndex == GPRs.size()) {
6714 const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
6715 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
6716 return false;
6717 }
6718
6719 // Corner case for 32-bit codegen. We have 2 registers to pass the first
6720 // half of the argument, and then need to pass the remaining half on the
6721 // stack.
6722 if (GPRs[NextRegIndex] == PPC::R9) {
6723 const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
6724 State.addLoc(
6725 CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo));
6726
6727 const unsigned FirstReg = State.AllocateReg(PPC::R9);
6728 const unsigned SecondReg = State.AllocateReg(PPC::R10);
6729 assert(FirstReg && SecondReg &&(static_cast <bool> (FirstReg && SecondReg &&
"Allocating R9 or R10 unexpectedly failed.") ? void (0) : __assert_fail
("FirstReg && SecondReg && \"Allocating R9 or R10 unexpectedly failed.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6730, __extension__ __PRETTY_FUNCTION__))
6730 "Allocating R9 or R10 unexpectedly failed.")(static_cast <bool> (FirstReg && SecondReg &&
"Allocating R9 or R10 unexpectedly failed.") ? void (0) : __assert_fail
("FirstReg && SecondReg && \"Allocating R9 or R10 unexpectedly failed.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6730, __extension__ __PRETTY_FUNCTION__))
;
6731 State.addLoc(
6732 CCValAssign::getCustomReg(ValNo, ValVT, FirstReg, RegVT, LocInfo));
6733 State.addLoc(
6734 CCValAssign::getCustomReg(ValNo, ValVT, SecondReg, RegVT, LocInfo));
6735 return false;
6736 }
6737
6738 // We have enough GPRs to fully pass the vector argument, and we have
6739 // already consumed any underaligned registers. Start with the custom
6740 // MemLoc and then the custom RegLocs.
6741 const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
6742 State.addLoc(
6743 CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo));
6744 for (unsigned I = 0; I != VecSize; I += PtrSize) {
6745 const unsigned Reg = State.AllocateReg(GPRs);
6746 assert(Reg && "Failed to allocated register for vararg vector argument")(static_cast <bool> (Reg && "Failed to allocated register for vararg vector argument"
) ? void (0) : __assert_fail ("Reg && \"Failed to allocated register for vararg vector argument\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6746, __extension__ __PRETTY_FUNCTION__))
;
6747 State.addLoc(
6748 CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo));
6749 }
6750 return false;
6751 }
6752 }
6753 return true;
6754}
6755
6756// So far, this function is only used by LowerFormalArguments_AIX()
6757static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT,
6758 bool IsPPC64,
6759 bool HasP8Vector,
6760 bool HasVSX) {
6761 assert((IsPPC64 || SVT != MVT::i64) &&(static_cast <bool> ((IsPPC64 || SVT != MVT::i64) &&
"i64 should have been split for 32-bit codegen.") ? void (0)
: __assert_fail ("(IsPPC64 || SVT != MVT::i64) && \"i64 should have been split for 32-bit codegen.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6762, __extension__ __PRETTY_FUNCTION__))
6762 "i64 should have been split for 32-bit codegen.")(static_cast <bool> ((IsPPC64 || SVT != MVT::i64) &&
"i64 should have been split for 32-bit codegen.") ? void (0)
: __assert_fail ("(IsPPC64 || SVT != MVT::i64) && \"i64 should have been split for 32-bit codegen.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6762, __extension__ __PRETTY_FUNCTION__))
;
6763
6764 switch (SVT) {
6765 default:
6766 report_fatal_error("Unexpected value type for formal argument");
6767 case MVT::i1:
6768 case MVT::i32:
6769 case MVT::i64:
6770 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
6771 case MVT::f32:
6772 return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass;
6773 case MVT::f64:
6774 return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass;
6775 case MVT::v4f32:
6776 case MVT::v4i32:
6777 case MVT::v8i16:
6778 case MVT::v16i8:
6779 case MVT::v2i64:
6780 case MVT::v2f64:
6781 case MVT::v1i128:
6782 return &PPC::VRRCRegClass;
6783 }
6784}
6785
6786static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
6787 SelectionDAG &DAG, SDValue ArgValue,
6788 MVT LocVT, const SDLoc &dl) {
6789 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger())(static_cast <bool> (ValVT.isScalarInteger() &&
LocVT.isScalarInteger()) ? void (0) : __assert_fail ("ValVT.isScalarInteger() && LocVT.isScalarInteger()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6789, __extension__ __PRETTY_FUNCTION__))
;
6790 assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())(static_cast <bool> (ValVT.getFixedSizeInBits() < LocVT
.getFixedSizeInBits()) ? void (0) : __assert_fail ("ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6790, __extension__ __PRETTY_FUNCTION__))
;
6791
6792 if (Flags.isSExt())
6793 ArgValue = DAG.getNode(ISD::AssertSext, dl, LocVT, ArgValue,
6794 DAG.getValueType(ValVT));
6795 else if (Flags.isZExt())
6796 ArgValue = DAG.getNode(ISD::AssertZext, dl, LocVT, ArgValue,
6797 DAG.getValueType(ValVT));
6798
6799 return DAG.getNode(ISD::TRUNCATE, dl, ValVT, ArgValue);
6800}
6801
6802static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
6803 const unsigned LASize = FL->getLinkageSize();
6804
6805 if (PPC::GPRCRegClass.contains(Reg)) {
6806 assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&(static_cast <bool> (Reg >= PPC::R3 && Reg <=
PPC::R10 && "Reg must be a valid argument register!"
) ? void (0) : __assert_fail ("Reg >= PPC::R3 && Reg <= PPC::R10 && \"Reg must be a valid argument register!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6807, __extension__ __PRETTY_FUNCTION__))
6807 "Reg must be a valid argument register!")(static_cast <bool> (Reg >= PPC::R3 && Reg <=
PPC::R10 && "Reg must be a valid argument register!"
) ? void (0) : __assert_fail ("Reg >= PPC::R3 && Reg <= PPC::R10 && \"Reg must be a valid argument register!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6807, __extension__ __PRETTY_FUNCTION__))
;
6808 return LASize + 4 * (Reg - PPC::R3);
6809 }
6810
6811 if (PPC::G8RCRegClass.contains(Reg)) {
6812 assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&(static_cast <bool> (Reg >= PPC::X3 && Reg <=
PPC::X10 && "Reg must be a valid argument register!"
) ? void (0) : __assert_fail ("Reg >= PPC::X3 && Reg <= PPC::X10 && \"Reg must be a valid argument register!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6813, __extension__ __PRETTY_FUNCTION__))
6813 "Reg must be a valid argument register!")(static_cast <bool> (Reg >= PPC::X3 && Reg <=
PPC::X10 && "Reg must be a valid argument register!"
) ? void (0) : __assert_fail ("Reg >= PPC::X3 && Reg <= PPC::X10 && \"Reg must be a valid argument register!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6813, __extension__ __PRETTY_FUNCTION__))
;
6814 return LASize + 8 * (Reg - PPC::X3);
6815 }
6816
6817 llvm_unreachable("Only general purpose registers expected.")::llvm::llvm_unreachable_internal("Only general purpose registers expected."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6817)
;
6818}
6819
6820// AIX ABI Stack Frame Layout:
6821//
6822// Low Memory +--------------------------------------------+
6823// SP +---> | Back chain | ---+
6824// | +--------------------------------------------+ |
6825// | | Saved Condition Register | |
6826// | +--------------------------------------------+ |
6827// | | Saved Linkage Register | |
6828// | +--------------------------------------------+ | Linkage Area
6829// | | Reserved for compilers | |
6830// | +--------------------------------------------+ |
6831// | | Reserved for binders | |
6832// | +--------------------------------------------+ |
6833// | | Saved TOC pointer | ---+
6834// | +--------------------------------------------+
6835// | | Parameter save area |
6836// | +--------------------------------------------+
6837// | | Alloca space |
6838// | +--------------------------------------------+
6839// | | Local variable space |
6840// | +--------------------------------------------+
6841// | | Float/int conversion temporary |
6842// | +--------------------------------------------+
6843// | | Save area for AltiVec registers |
6844// | +--------------------------------------------+
6845// | | AltiVec alignment padding |
6846// | +--------------------------------------------+
6847// | | Save area for VRSAVE register |
6848// | +--------------------------------------------+
6849// | | Save area for General Purpose registers |
6850// | +--------------------------------------------+
6851// | | Save area for Floating Point registers |
6852// | +--------------------------------------------+
6853// +---- | Back chain |
6854// High Memory +--------------------------------------------+
6855//
6856// Specifications:
6857// AIX 7.2 Assembler Language Reference
6858// Subroutine linkage convention
6859
6860SDValue PPCTargetLowering::LowerFormalArguments_AIX(
6861 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
6862 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
6863 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
6864
6865 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold ||(static_cast <bool> ((CallConv == CallingConv::C || CallConv
== CallingConv::Cold || CallConv == CallingConv::Fast) &&
"Unexpected calling convention!") ? void (0) : __assert_fail
("(CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6867, __extension__ __PRETTY_FUNCTION__))
6866 CallConv == CallingConv::Fast) &&(static_cast <bool> ((CallConv == CallingConv::C || CallConv
== CallingConv::Cold || CallConv == CallingConv::Fast) &&
"Unexpected calling convention!") ? void (0) : __assert_fail
("(CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6867, __extension__ __PRETTY_FUNCTION__))
6867 "Unexpected calling convention!")(static_cast <bool> ((CallConv == CallingConv::C || CallConv
== CallingConv::Cold || CallConv == CallingConv::Fast) &&
"Unexpected calling convention!") ? void (0) : __assert_fail
("(CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6867, __extension__ __PRETTY_FUNCTION__))
;
6868
6869 if (getTargetMachine().Options.GuaranteedTailCallOpt)
6870 report_fatal_error("Tail call support is unimplemented on AIX.");
6871
6872 if (useSoftFloat())
6873 report_fatal_error("Soft float support is unimplemented on AIX.");
6874
6875 const PPCSubtarget &Subtarget =
6876 static_cast<const PPCSubtarget &>(DAG.getSubtarget());
6877
6878 const bool IsPPC64 = Subtarget.isPPC64();
6879 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
6880
6881 // Assign locations to all of the incoming arguments.
6882 SmallVector<CCValAssign, 16> ArgLocs;
6883 MachineFunction &MF = DAG.getMachineFunction();
6884 MachineFrameInfo &MFI = MF.getFrameInfo();
6885 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
6886 AIXCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
6887
6888 const EVT PtrVT = getPointerTy(MF.getDataLayout());
6889 // Reserve space for the linkage area on the stack.
6890 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
6891 CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize));
6892 CCInfo.AnalyzeFormalArguments(Ins, CC_AIX);
6893
6894 SmallVector<SDValue, 8> MemOps;
6895
6896 for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) {
6897 CCValAssign &VA = ArgLocs[I++];
6898 MVT LocVT = VA.getLocVT();
6899 MVT ValVT = VA.getValVT();
6900 ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
6901 // For compatibility with the AIX XL compiler, the float args in the
6902 // parameter save area are initialized even if the argument is available
6903 // in register. The caller is required to initialize both the register
6904 // and memory, however, the callee can choose to expect it in either.
6905 // The memloc is dismissed here because the argument is retrieved from
6906 // the register.
6907 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint())
6908 continue;
6909
6910 auto HandleMemLoc = [&]() {
6911 const unsigned LocSize = LocVT.getStoreSize();
6912 const unsigned ValSize = ValVT.getStoreSize();
6913 assert((ValSize <= LocSize) &&(static_cast <bool> ((ValSize <= LocSize) &&
"Object size is larger than size of MemLoc") ? void (0) : __assert_fail
("(ValSize <= LocSize) && \"Object size is larger than size of MemLoc\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6914, __extension__ __PRETTY_FUNCTION__))
6914 "Object size is larger than size of MemLoc")(static_cast <bool> ((ValSize <= LocSize) &&
"Object size is larger than size of MemLoc") ? void (0) : __assert_fail
("(ValSize <= LocSize) && \"Object size is larger than size of MemLoc\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6914, __extension__ __PRETTY_FUNCTION__))
;
6915 int CurArgOffset = VA.getLocMemOffset();
6916 // Objects are right-justified because AIX is big-endian.
6917 if (LocSize > ValSize)
6918 CurArgOffset += LocSize - ValSize;
6919 // Potential tail calls could cause overwriting of argument stack slots.
6920 const bool IsImmutable =
6921 !(getTargetMachine().Options.GuaranteedTailCallOpt &&
6922 (CallConv == CallingConv::Fast));
6923 int FI = MFI.CreateFixedObject(ValSize, CurArgOffset, IsImmutable);
6924 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
6925 SDValue ArgValue =
6926 DAG.getLoad(ValVT, dl, Chain, FIN, MachinePointerInfo());
6927 InVals.push_back(ArgValue);
6928 };
6929
6930 // Vector arguments to VaArg functions are passed both on the stack, and
6931 // in any available GPRs. Load the value from the stack and add the GPRs
6932 // as live ins.
6933 if (VA.isMemLoc() && VA.needsCustom()) {
6934 assert(ValVT.isVector() && "Unexpected Custom MemLoc type.")(static_cast <bool> (ValVT.isVector() && "Unexpected Custom MemLoc type."
) ? void (0) : __assert_fail ("ValVT.isVector() && \"Unexpected Custom MemLoc type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6934, __extension__ __PRETTY_FUNCTION__))
;
6935 assert(isVarArg && "Only use custom memloc for vararg.")(static_cast <bool> (isVarArg && "Only use custom memloc for vararg."
) ? void (0) : __assert_fail ("isVarArg && \"Only use custom memloc for vararg.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6935, __extension__ __PRETTY_FUNCTION__))
;
6936 // ValNo of the custom MemLoc, so we can compare it to the ValNo of the
6937 // matching custom RegLocs.
6938 const unsigned OriginalValNo = VA.getValNo();
6939 (void)OriginalValNo;
6940
6941 auto HandleCustomVecRegLoc = [&]() {
6942 assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&(static_cast <bool> (I != End && ArgLocs[I].isRegLoc
() && ArgLocs[I].needsCustom() && "Missing custom RegLoc."
) ? void (0) : __assert_fail ("I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && \"Missing custom RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6943, __extension__ __PRETTY_FUNCTION__))
6943 "Missing custom RegLoc.")(static_cast <bool> (I != End && ArgLocs[I].isRegLoc
() && ArgLocs[I].needsCustom() && "Missing custom RegLoc."
) ? void (0) : __assert_fail ("I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && \"Missing custom RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6943, __extension__ __PRETTY_FUNCTION__))
;
6944 VA = ArgLocs[I++];
6945 assert(VA.getValVT().isVector() &&(static_cast <bool> (VA.getValVT().isVector() &&
"Unexpected Val type for custom RegLoc.") ? void (0) : __assert_fail
("VA.getValVT().isVector() && \"Unexpected Val type for custom RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6946, __extension__ __PRETTY_FUNCTION__))
6946 "Unexpected Val type for custom RegLoc.")(static_cast <bool> (VA.getValVT().isVector() &&
"Unexpected Val type for custom RegLoc.") ? void (0) : __assert_fail
("VA.getValVT().isVector() && \"Unexpected Val type for custom RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6946, __extension__ __PRETTY_FUNCTION__))
;
6947 assert(VA.getValNo() == OriginalValNo &&(static_cast <bool> (VA.getValNo() == OriginalValNo &&
"ValNo mismatch between custom MemLoc and RegLoc.") ? void (
0) : __assert_fail ("VA.getValNo() == OriginalValNo && \"ValNo mismatch between custom MemLoc and RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6948, __extension__ __PRETTY_FUNCTION__))
6948 "ValNo mismatch between custom MemLoc and RegLoc.")(static_cast <bool> (VA.getValNo() == OriginalValNo &&
"ValNo mismatch between custom MemLoc and RegLoc.") ? void (
0) : __assert_fail ("VA.getValNo() == OriginalValNo && \"ValNo mismatch between custom MemLoc and RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6948, __extension__ __PRETTY_FUNCTION__))
;
6949 MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy;
6950 MF.addLiveIn(VA.getLocReg(),
6951 getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(),
6952 Subtarget.hasVSX()));
6953 };
6954
6955 HandleMemLoc();
6956 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
6957 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
6958 // R10.
6959 HandleCustomVecRegLoc();
6960 HandleCustomVecRegLoc();
6961
6962 // If we are targeting 32-bit, there might be 2 extra custom RegLocs if
6963 // we passed the vector in R5, R6, R7 and R8.
6964 if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) {
6965 assert(!IsPPC64 &&(static_cast <bool> (!IsPPC64 && "Only 2 custom RegLocs expected for 64-bit codegen."
) ? void (0) : __assert_fail ("!IsPPC64 && \"Only 2 custom RegLocs expected for 64-bit codegen.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6966, __extension__ __PRETTY_FUNCTION__))
6966 "Only 2 custom RegLocs expected for 64-bit codegen.")(static_cast <bool> (!IsPPC64 && "Only 2 custom RegLocs expected for 64-bit codegen."
) ? void (0) : __assert_fail ("!IsPPC64 && \"Only 2 custom RegLocs expected for 64-bit codegen.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 6966, __extension__ __PRETTY_FUNCTION__))
;
6967 HandleCustomVecRegLoc();
6968 HandleCustomVecRegLoc();
6969 }
6970
6971 continue;
6972 }
6973
6974 if (VA.isRegLoc()) {
6975 if (VA.getValVT().isScalarInteger())
6976 FuncInfo->appendParameterType(PPCFunctionInfo::FixedType);
6977 else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) {
6978 switch (VA.getValVT().SimpleTy) {
6979 default:
6980 report_fatal_error("Unhandled value type for argument.");
6981 case MVT::f32:
6982 FuncInfo->appendParameterType(PPCFunctionInfo::ShortFloatingPoint);
6983 break;
6984 case MVT::f64:
6985 FuncInfo->appendParameterType(PPCFunctionInfo::LongFloatingPoint);
6986 break;
6987 }
6988 } else if (VA.getValVT().isVector()) {
6989 switch (VA.getValVT().SimpleTy) {
6990 default:
6991 report_fatal_error("Unhandled value type for argument.");
6992 case MVT::v16i8:
6993 FuncInfo->appendParameterType(PPCFunctionInfo::VectorChar);
6994 break;
6995 case MVT::v8i16:
6996 FuncInfo->appendParameterType(PPCFunctionInfo::VectorShort);
6997 break;
6998 case MVT::v4i32:
6999 case MVT::v2i64:
7000 case MVT::v1i128:
7001 FuncInfo->appendParameterType(PPCFunctionInfo::VectorInt);
7002 break;
7003 case MVT::v4f32:
7004 case MVT::v2f64:
7005 FuncInfo->appendParameterType(PPCFunctionInfo::VectorFloat);
7006 break;
7007 }
7008 }
7009 }
7010
7011 if (Flags.isByVal() && VA.isMemLoc()) {
7012 const unsigned Size =
7013 alignTo(Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
7014 PtrByteSize);
7015 const int FI = MF.getFrameInfo().CreateFixedObject(
7016 Size, VA.getLocMemOffset(), /* IsImmutable */ false,
7017 /* IsAliased */ true);
7018 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
7019 InVals.push_back(FIN);
7020
7021 continue;
7022 }
7023
7024 if (Flags.isByVal()) {
7025 assert(VA.isRegLoc() && "MemLocs should already be handled.")(static_cast <bool> (VA.isRegLoc() && "MemLocs should already be handled."
) ? void (0) : __assert_fail ("VA.isRegLoc() && \"MemLocs should already be handled.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7025, __extension__ __PRETTY_FUNCTION__))
;
7026
7027 const MCPhysReg ArgReg = VA.getLocReg();
7028 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
7029
7030 if (Flags.getNonZeroByValAlign() > PtrByteSize)
7031 report_fatal_error("Over aligned byvals not supported yet.");
7032
7033 const unsigned StackSize = alignTo(Flags.getByValSize(), PtrByteSize);
7034 const int FI = MF.getFrameInfo().CreateFixedObject(
7035 StackSize, mapArgRegToOffsetAIX(ArgReg, FL), /* IsImmutable */ false,
7036 /* IsAliased */ true);
7037 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
7038 InVals.push_back(FIN);
7039
7040 // Add live ins for all the RegLocs for the same ByVal.
7041 const TargetRegisterClass *RegClass =
7042 IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7043
7044 auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg,
7045 unsigned Offset) {
7046 const unsigned VReg = MF.addLiveIn(PhysReg, RegClass);
7047 // Since the callers side has left justified the aggregate in the
7048 // register, we can simply store the entire register into the stack
7049 // slot.
7050 SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, VReg, LocVT);
7051 // The store to the fixedstack object is needed becuase accessing a
7052 // field of the ByVal will use a gep and load. Ideally we will optimize
7053 // to extracting the value from the register directly, and elide the
7054 // stores when the arguments address is not taken, but that will need to
7055 // be future work.
7056 SDValue Store = DAG.getStore(
7057 CopyFrom.getValue(1), dl, CopyFrom,
7058 DAG.getObjectPtrOffset(dl, FIN, TypeSize::Fixed(Offset)),
7059 MachinePointerInfo::getFixedStack(MF, FI, Offset));
7060
7061 MemOps.push_back(Store);
7062 };
7063
7064 unsigned Offset = 0;
7065 HandleRegLoc(VA.getLocReg(), Offset);
7066 Offset += PtrByteSize;
7067 for (; Offset != StackSize && ArgLocs[I].isRegLoc();
7068 Offset += PtrByteSize) {
7069 assert(ArgLocs[I].getValNo() == VA.getValNo() &&(static_cast <bool> (ArgLocs[I].getValNo() == VA.getValNo
() && "RegLocs should be for ByVal argument.") ? void
(0) : __assert_fail ("ArgLocs[I].getValNo() == VA.getValNo() && \"RegLocs should be for ByVal argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7070, __extension__ __PRETTY_FUNCTION__))
7070 "RegLocs should be for ByVal argument.")(static_cast <bool> (ArgLocs[I].getValNo() == VA.getValNo
() && "RegLocs should be for ByVal argument.") ? void
(0) : __assert_fail ("ArgLocs[I].getValNo() == VA.getValNo() && \"RegLocs should be for ByVal argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7070, __extension__ __PRETTY_FUNCTION__))
;
7071
7072 const CCValAssign RL = ArgLocs[I++];
7073 HandleRegLoc(RL.getLocReg(), Offset);
7074 FuncInfo->appendParameterType(PPCFunctionInfo::FixedType);
7075 }
7076
7077 if (Offset != StackSize) {
7078 assert(ArgLocs[I].getValNo() == VA.getValNo() &&(static_cast <bool> (ArgLocs[I].getValNo() == VA.getValNo
() && "Expected MemLoc for remaining bytes.") ? void (
0) : __assert_fail ("ArgLocs[I].getValNo() == VA.getValNo() && \"Expected MemLoc for remaining bytes.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7079, __extension__ __PRETTY_FUNCTION__))
7079 "Expected MemLoc for remaining bytes.")(static_cast <bool> (ArgLocs[I].getValNo() == VA.getValNo
() && "Expected MemLoc for remaining bytes.") ? void (
0) : __assert_fail ("ArgLocs[I].getValNo() == VA.getValNo() && \"Expected MemLoc for remaining bytes.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7079, __extension__ __PRETTY_FUNCTION__))
;
7080 assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes.")(static_cast <bool> (ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes."
) ? void (0) : __assert_fail ("ArgLocs[I].isMemLoc() && \"Expected MemLoc for remaining bytes.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7080, __extension__ __PRETTY_FUNCTION__))
;
7081 // Consume the MemLoc.The InVal has already been emitted, so nothing
7082 // more needs to be done.
7083 ++I;
7084 }
7085
7086 continue;
7087 }
7088
7089 if (VA.isRegLoc() && !VA.needsCustom()) {
7090 MVT::SimpleValueType SVT = ValVT.SimpleTy;
7091 Register VReg =
7092 MF.addLiveIn(VA.getLocReg(),
7093 getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(),
7094 Subtarget.hasVSX()));
7095 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT);
7096 if (ValVT.isScalarInteger() &&
7097 (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) {
7098 ArgValue =
7099 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl);
7100 }
7101 InVals.push_back(ArgValue);
7102 continue;
7103 }
7104 if (VA.isMemLoc()) {
7105 HandleMemLoc();
7106 continue;
7107 }
7108 }
7109
7110 // On AIX a minimum of 8 words is saved to the parameter save area.
7111 const unsigned MinParameterSaveArea = 8 * PtrByteSize;
7112 // Area that is at least reserved in the caller of this function.
7113 unsigned CallerReservedArea =
7114 std::max(CCInfo.getNextStackOffset(), LinkageSize + MinParameterSaveArea);
7115
7116 // Set the size that is at least reserved in caller of this function. Tail
7117 // call optimized function's reserved stack space needs to be aligned so
7118 // that taking the difference between two stack areas will result in an
7119 // aligned stack.
7120 CallerReservedArea =
7121 EnsureStackAlignment(Subtarget.getFrameLowering(), CallerReservedArea);
7122 FuncInfo->setMinReservedArea(CallerReservedArea);
7123
7124 if (isVarArg) {
7125 FuncInfo->setVarArgsFrameIndex(
7126 MFI.CreateFixedObject(PtrByteSize, CCInfo.getNextStackOffset(), true));
7127 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
7128
7129 static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6,
7130 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
7131
7132 static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6,
7133 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
7134 const unsigned NumGPArgRegs = array_lengthof(IsPPC64 ? GPR_64 : GPR_32);
7135
7136 // The fixed integer arguments of a variadic function are stored to the
7137 // VarArgsFrameIndex on the stack so that they may be loaded by
7138 // dereferencing the result of va_next.
7139 for (unsigned GPRIndex =
7140 (CCInfo.getNextStackOffset() - LinkageSize) / PtrByteSize;
7141 GPRIndex < NumGPArgRegs; ++GPRIndex) {
7142
7143 const unsigned VReg =
7144 IsPPC64 ? MF.addLiveIn(GPR_64[GPRIndex], &PPC::G8RCRegClass)
7145 : MF.addLiveIn(GPR_32[GPRIndex], &PPC::GPRCRegClass);
7146
7147 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
7148 SDValue Store =
7149 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
7150 MemOps.push_back(Store);
7151 // Increment the address for the next argument to store.
7152 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
7153 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
7154 }
7155 }
7156
7157 if (!MemOps.empty())
7158 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
7159
7160 return Chain;
7161}
7162
7163SDValue PPCTargetLowering::LowerCall_AIX(
7164 SDValue Chain, SDValue Callee, CallFlags CFlags,
7165 const SmallVectorImpl<ISD::OutputArg> &Outs,
7166 const SmallVectorImpl<SDValue> &OutVals,
7167 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7168 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
7169 const CallBase *CB) const {
7170 // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
7171 // AIX ABI stack frame layout.
7172
7173 assert((CFlags.CallConv == CallingConv::C ||(static_cast <bool> ((CFlags.CallConv == CallingConv::C
|| CFlags.CallConv == CallingConv::Cold || CFlags.CallConv ==
CallingConv::Fast) && "Unexpected calling convention!"
) ? void (0) : __assert_fail ("(CFlags.CallConv == CallingConv::C || CFlags.CallConv == CallingConv::Cold || CFlags.CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7176, __extension__ __PRETTY_FUNCTION__))
7174 CFlags.CallConv == CallingConv::Cold ||(static_cast <bool> ((CFlags.CallConv == CallingConv::C
|| CFlags.CallConv == CallingConv::Cold || CFlags.CallConv ==
CallingConv::Fast) && "Unexpected calling convention!"
) ? void (0) : __assert_fail ("(CFlags.CallConv == CallingConv::C || CFlags.CallConv == CallingConv::Cold || CFlags.CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7176, __extension__ __PRETTY_FUNCTION__))
7175 CFlags.CallConv == CallingConv::Fast) &&(static_cast <bool> ((CFlags.CallConv == CallingConv::C
|| CFlags.CallConv == CallingConv::Cold || CFlags.CallConv ==
CallingConv::Fast) && "Unexpected calling convention!"
) ? void (0) : __assert_fail ("(CFlags.CallConv == CallingConv::C || CFlags.CallConv == CallingConv::Cold || CFlags.CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7176, __extension__ __PRETTY_FUNCTION__))
7176 "Unexpected calling convention!")(static_cast <bool> ((CFlags.CallConv == CallingConv::C
|| CFlags.CallConv == CallingConv::Cold || CFlags.CallConv ==
CallingConv::Fast) && "Unexpected calling convention!"
) ? void (0) : __assert_fail ("(CFlags.CallConv == CallingConv::C || CFlags.CallConv == CallingConv::Cold || CFlags.CallConv == CallingConv::Fast) && \"Unexpected calling convention!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7176, __extension__ __PRETTY_FUNCTION__))
;
7177
7178 if (CFlags.IsPatchPoint)
7179 report_fatal_error("This call type is unimplemented on AIX.");
7180
7181 const PPCSubtarget& Subtarget =
7182 static_cast<const PPCSubtarget&>(DAG.getSubtarget());
7183
7184 MachineFunction &MF = DAG.getMachineFunction();
7185 SmallVector<CCValAssign, 16> ArgLocs;
7186 AIXCCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
7187 *DAG.getContext());
7188
7189 // Reserve space for the linkage save area (LSA) on the stack.
7190 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
7191 // [SP][CR][LR][2 x reserved][TOC].
7192 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
7193 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7194 const bool IsPPC64 = Subtarget.isPPC64();
7195 const EVT PtrVT = getPointerTy(DAG.getDataLayout());
7196 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7197 CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize));
7198 CCInfo.AnalyzeCallOperands(Outs, CC_AIX);
7199
7200 // The prolog code of the callee may store up to 8 GPR argument registers to
7201 // the stack, allowing va_start to index over them in memory if the callee
7202 // is variadic.
7203 // Because we cannot tell if this is needed on the caller side, we have to
7204 // conservatively assume that it is needed. As such, make sure we have at
7205 // least enough stack space for the caller to store the 8 GPRs.
7206 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize;
7207 const unsigned NumBytes = std::max(LinkageSize + MinParameterSaveAreaSize,
7208 CCInfo.getNextStackOffset());
7209
7210 // Adjust the stack pointer for the new arguments...
7211 // These operations are automatically eliminated by the prolog/epilog pass.
7212 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
7213 SDValue CallSeqStart = Chain;
7214
7215 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
7216 SmallVector<SDValue, 8> MemOpChains;
7217
7218 // Set up a copy of the stack pointer for loading and storing any
7219 // arguments that may not fit in the registers available for argument
7220 // passing.
7221 const SDValue StackPtr = IsPPC64 ? DAG.getRegister(PPC::X1, MVT::i64)
7222 : DAG.getRegister(PPC::R1, MVT::i32);
7223
7224 for (unsigned I = 0, E = ArgLocs.size(); I != E;) {
7225 const unsigned ValNo = ArgLocs[I].getValNo();
7226 SDValue Arg = OutVals[ValNo];
7227 ISD::ArgFlagsTy Flags = Outs[ValNo].Flags;
7228
7229 if (Flags.isByVal()) {
7230 const unsigned ByValSize = Flags.getByValSize();
7231
7232 // Nothing to do for zero-sized ByVals on the caller side.
7233 if (!ByValSize) {
7234 ++I;
7235 continue;
7236 }
7237
7238 auto GetLoad = [&](EVT VT, unsigned LoadOffset) {
7239 return DAG.getExtLoad(
7240 ISD::ZEXTLOAD, dl, PtrVT, Chain,
7241 (LoadOffset != 0)
7242 ? DAG.getObjectPtrOffset(dl, Arg, TypeSize::Fixed(LoadOffset))
7243 : Arg,
7244 MachinePointerInfo(), VT);
7245 };
7246
7247 unsigned LoadOffset = 0;
7248
7249 // Initialize registers, which are fully occupied by the by-val argument.
7250 while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) {
7251 SDValue Load = GetLoad(PtrVT, LoadOffset);
7252 MemOpChains.push_back(Load.getValue(1));
7253 LoadOffset += PtrByteSize;
7254 const CCValAssign &ByValVA = ArgLocs[I++];
7255 assert(ByValVA.getValNo() == ValNo &&(static_cast <bool> (ByValVA.getValNo() == ValNo &&
"Unexpected location for pass-by-value argument.") ? void (0
) : __assert_fail ("ByValVA.getValNo() == ValNo && \"Unexpected location for pass-by-value argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7256, __extension__ __PRETTY_FUNCTION__))
7256 "Unexpected location for pass-by-value argument.")(static_cast <bool> (ByValVA.getValNo() == ValNo &&
"Unexpected location for pass-by-value argument.") ? void (0
) : __assert_fail ("ByValVA.getValNo() == ValNo && \"Unexpected location for pass-by-value argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7256, __extension__ __PRETTY_FUNCTION__))
;
7257 RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), Load));
7258 }
7259
7260 if (LoadOffset == ByValSize)
7261 continue;
7262
7263 // There must be one more loc to handle the remainder.
7264 assert(ArgLocs[I].getValNo() == ValNo &&(static_cast <bool> (ArgLocs[I].getValNo() == ValNo &&
"Expected additional location for by-value argument.") ? void
(0) : __assert_fail ("ArgLocs[I].getValNo() == ValNo && \"Expected additional location for by-value argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7265, __extension__ __PRETTY_FUNCTION__))
7265 "Expected additional location for by-value argument.")(static_cast <bool> (ArgLocs[I].getValNo() == ValNo &&
"Expected additional location for by-value argument.") ? void
(0) : __assert_fail ("ArgLocs[I].getValNo() == ValNo && \"Expected additional location for by-value argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7265, __extension__ __PRETTY_FUNCTION__))
;
7266
7267 if (ArgLocs[I].isMemLoc()) {
7268 assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg.")(static_cast <bool> (LoadOffset < ByValSize &&
"Unexpected memloc for by-val arg.") ? void (0) : __assert_fail
("LoadOffset < ByValSize && \"Unexpected memloc for by-val arg.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7268, __extension__ __PRETTY_FUNCTION__))
;
7269 const CCValAssign &ByValVA = ArgLocs[I++];
7270 ISD::ArgFlagsTy MemcpyFlags = Flags;
7271 // Only memcpy the bytes that don't pass in register.
7272 MemcpyFlags.setByValSize(ByValSize - LoadOffset);
7273 Chain = CallSeqStart = createMemcpyOutsideCallSeq(
7274 (LoadOffset != 0)
7275 ? DAG.getObjectPtrOffset(dl, Arg, TypeSize::Fixed(LoadOffset))
7276 : Arg,
7277 DAG.getObjectPtrOffset(dl, StackPtr,
7278 TypeSize::Fixed(ByValVA.getLocMemOffset())),
7279 CallSeqStart, MemcpyFlags, DAG, dl);
7280 continue;
7281 }
7282
7283 // Initialize the final register residue.
7284 // Any residue that occupies the final by-val arg register must be
7285 // left-justified on AIX. Loads must be a power-of-2 size and cannot be
7286 // larger than the ByValSize. For example: a 7 byte by-val arg requires 4,
7287 // 2 and 1 byte loads.
7288 const unsigned ResidueBytes = ByValSize % PtrByteSize;
7289 assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize &&(static_cast <bool> (ResidueBytes != 0 && LoadOffset
+ PtrByteSize > ByValSize && "Unexpected register residue for by-value argument."
) ? void (0) : __assert_fail ("ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize && \"Unexpected register residue for by-value argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7290, __extension__ __PRETTY_FUNCTION__))
7290 "Unexpected register residue for by-value argument.")(static_cast <bool> (ResidueBytes != 0 && LoadOffset
+ PtrByteSize > ByValSize && "Unexpected register residue for by-value argument."
) ? void (0) : __assert_fail ("ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize && \"Unexpected register residue for by-value argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7290, __extension__ __PRETTY_FUNCTION__))
;
7291 SDValue ResidueVal;
7292 for (unsigned Bytes = 0; Bytes != ResidueBytes;) {
7293 const unsigned N = PowerOf2Floor(ResidueBytes - Bytes);
7294 const MVT VT =
7295 N == 1 ? MVT::i8
7296 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64));
7297 SDValue Load = GetLoad(VT, LoadOffset);
7298 MemOpChains.push_back(Load.getValue(1));
7299 LoadOffset += N;
7300 Bytes += N;
7301
7302 // By-val arguments are passed left-justfied in register.
7303 // Every load here needs to be shifted, otherwise a full register load
7304 // should have been used.
7305 assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) &&(static_cast <bool> (PtrVT.getSimpleVT().getSizeInBits(
) > (Bytes * 8) && "Unexpected load emitted during handling of pass-by-value "
"argument.") ? void (0) : __assert_fail ("PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) && \"Unexpected load emitted during handling of pass-by-value \" \"argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7307, __extension__ __PRETTY_FUNCTION__))
7306 "Unexpected load emitted during handling of pass-by-value "(static_cast <bool> (PtrVT.getSimpleVT().getSizeInBits(
) > (Bytes * 8) && "Unexpected load emitted during handling of pass-by-value "
"argument.") ? void (0) : __assert_fail ("PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) && \"Unexpected load emitted during handling of pass-by-value \" \"argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7307, __extension__ __PRETTY_FUNCTION__))
7307 "argument.")(static_cast <bool> (PtrVT.getSimpleVT().getSizeInBits(
) > (Bytes * 8) && "Unexpected load emitted during handling of pass-by-value "
"argument.") ? void (0) : __assert_fail ("PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) && \"Unexpected load emitted during handling of pass-by-value \" \"argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7307, __extension__ __PRETTY_FUNCTION__))
;
7308 unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8);
7309 EVT ShiftAmountTy =
7310 getShiftAmountTy(Load->getValueType(0), DAG.getDataLayout());
7311 SDValue SHLAmt = DAG.getConstant(NumSHLBits, dl, ShiftAmountTy);
7312 SDValue ShiftedLoad =
7313 DAG.getNode(ISD::SHL, dl, Load.getValueType(), Load, SHLAmt);
7314 ResidueVal = ResidueVal ? DAG.getNode(ISD::OR, dl, PtrVT, ResidueVal,
7315 ShiftedLoad)
7316 : ShiftedLoad;
7317 }
7318
7319 const CCValAssign &ByValVA = ArgLocs[I++];
7320 RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), ResidueVal));
7321 continue;
7322 }
7323
7324 CCValAssign &VA = ArgLocs[I++];
7325 const MVT LocVT = VA.getLocVT();
7326 const MVT ValVT = VA.getValVT();
7327
7328 switch (VA.getLocInfo()) {
7329 default:
7330 report_fatal_error("Unexpected argument extension type.");
7331 case CCValAssign::Full:
7332 break;
7333 case CCValAssign::ZExt:
7334 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
7335 break;
7336 case CCValAssign::SExt:
7337 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
7338 break;
7339 }
7340
7341 if (VA.isRegLoc() && !VA.needsCustom()) {
7342 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
7343 continue;
7344 }
7345
7346 // Vector arguments passed to VarArg functions need custom handling when
7347 // they are passed (at least partially) in GPRs.
7348 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) {
7349 assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args.")(static_cast <bool> (CFlags.IsVarArg && "Custom MemLocs only used for Vector args."
) ? void (0) : __assert_fail ("CFlags.IsVarArg && \"Custom MemLocs only used for Vector args.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7349, __extension__ __PRETTY_FUNCTION__))
;
7350 // Store value to its stack slot.
7351 SDValue PtrOff =
7352 DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType());
7353 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
7354 SDValue Store =
7355 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
7356 MemOpChains.push_back(Store);
7357 const unsigned OriginalValNo = VA.getValNo();
7358 // Then load the GPRs from the stack
7359 unsigned LoadOffset = 0;
7360 auto HandleCustomVecRegLoc = [&]() {
7361 assert(I != E && "Unexpected end of CCvalAssigns.")(static_cast <bool> (I != E && "Unexpected end of CCvalAssigns."
) ? void (0) : __assert_fail ("I != E && \"Unexpected end of CCvalAssigns.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7361, __extension__ __PRETTY_FUNCTION__))
;
7362 assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&(static_cast <bool> (ArgLocs[I].isRegLoc() && ArgLocs
[I].needsCustom() && "Expected custom RegLoc.") ? void
(0) : __assert_fail ("ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && \"Expected custom RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7363, __extension__ __PRETTY_FUNCTION__))
7363 "Expected custom RegLoc.")(static_cast <bool> (ArgLocs[I].isRegLoc() && ArgLocs
[I].needsCustom() && "Expected custom RegLoc.") ? void
(0) : __assert_fail ("ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && \"Expected custom RegLoc.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7363, __extension__ __PRETTY_FUNCTION__))
;
7364 CCValAssign RegVA = ArgLocs[I++];
7365 assert(RegVA.getValNo() == OriginalValNo &&(static_cast <bool> (RegVA.getValNo() == OriginalValNo &&
"Custom MemLoc ValNo and custom RegLoc ValNo must match.") ?
void (0) : __assert_fail ("RegVA.getValNo() == OriginalValNo && \"Custom MemLoc ValNo and custom RegLoc ValNo must match.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7366, __extension__ __PRETTY_FUNCTION__))
7366 "Custom MemLoc ValNo and custom RegLoc ValNo must match.")(static_cast <bool> (RegVA.getValNo() == OriginalValNo &&
"Custom MemLoc ValNo and custom RegLoc ValNo must match.") ?
void (0) : __assert_fail ("RegVA.getValNo() == OriginalValNo && \"Custom MemLoc ValNo and custom RegLoc ValNo must match.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7366, __extension__ __PRETTY_FUNCTION__))
;
7367 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
7368 DAG.getConstant(LoadOffset, dl, PtrVT));
7369 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Add, MachinePointerInfo());
7370 MemOpChains.push_back(Load.getValue(1));
7371 RegsToPass.push_back(std::make_pair(RegVA.getLocReg(), Load));
7372 LoadOffset += PtrByteSize;
7373 };
7374
7375 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7376 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7377 // R10.
7378 HandleCustomVecRegLoc();
7379 HandleCustomVecRegLoc();
7380
7381 if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7382 ArgLocs[I].getValNo() == OriginalValNo) {
7383 assert(!IsPPC64 &&(static_cast <bool> (!IsPPC64 && "Only 2 custom RegLocs expected for 64-bit codegen."
) ? void (0) : __assert_fail ("!IsPPC64 && \"Only 2 custom RegLocs expected for 64-bit codegen.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7384, __extension__ __PRETTY_FUNCTION__))
7384 "Only 2 custom RegLocs expected for 64-bit codegen.")(static_cast <bool> (!IsPPC64 && "Only 2 custom RegLocs expected for 64-bit codegen."
) ? void (0) : __assert_fail ("!IsPPC64 && \"Only 2 custom RegLocs expected for 64-bit codegen.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7384, __extension__ __PRETTY_FUNCTION__))
;
7385 HandleCustomVecRegLoc();
7386 HandleCustomVecRegLoc();
7387 }
7388
7389 continue;
7390 }
7391
7392 if (VA.isMemLoc()) {
7393 SDValue PtrOff =
7394 DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType());
7395 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
7396 MemOpChains.push_back(
7397 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()));
7398
7399 continue;
7400 }
7401
7402 if (!ValVT.isFloatingPoint())
7403 report_fatal_error(
7404 "Unexpected register handling for calling convention.");
7405
7406 // Custom handling is used for GPR initializations for vararg float
7407 // arguments.
7408 assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg &&(static_cast <bool> (VA.isRegLoc() && VA.needsCustom
() && CFlags.IsVarArg && LocVT.isInteger() &&
"Custom register handling only expected for VarArg.") ? void
(0) : __assert_fail ("VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg && LocVT.isInteger() && \"Custom register handling only expected for VarArg.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7410, __extension__ __PRETTY_FUNCTION__))
7409 LocVT.isInteger() &&(static_cast <bool> (VA.isRegLoc() && VA.needsCustom
() && CFlags.IsVarArg && LocVT.isInteger() &&
"Custom register handling only expected for VarArg.") ? void
(0) : __assert_fail ("VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg && LocVT.isInteger() && \"Custom register handling only expected for VarArg.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7410, __extension__ __PRETTY_FUNCTION__))
7410 "Custom register handling only expected for VarArg.")(static_cast <bool> (VA.isRegLoc() && VA.needsCustom
() && CFlags.IsVarArg && LocVT.isInteger() &&
"Custom register handling only expected for VarArg.") ? void
(0) : __assert_fail ("VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg && LocVT.isInteger() && \"Custom register handling only expected for VarArg.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7410, __extension__ __PRETTY_FUNCTION__))
;
7411
7412 SDValue ArgAsInt =
7413 DAG.getBitcast(MVT::getIntegerVT(ValVT.getSizeInBits()), Arg);
7414
7415 if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize())
7416 // f32 in 32-bit GPR
7417 // f64 in 64-bit GPR
7418 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgAsInt));
7419 else if (Arg.getValueType().getFixedSizeInBits() <
7420 LocVT.getFixedSizeInBits())
7421 // f32 in 64-bit GPR.
7422 RegsToPass.push_back(std::make_pair(
7423 VA.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, LocVT)));
7424 else {
7425 // f64 in two 32-bit GPRs
7426 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs.
7427 assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 &&(static_cast <bool> (Arg.getValueType() == MVT::f64 &&
CFlags.IsVarArg && !IsPPC64 && "Unexpected custom register for argument!"
) ? void (0) : __assert_fail ("Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 && \"Unexpected custom register for argument!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7428, __extension__ __PRETTY_FUNCTION__))
7428 "Unexpected custom register for argument!")(static_cast <bool> (Arg.getValueType() == MVT::f64 &&
CFlags.IsVarArg && !IsPPC64 && "Unexpected custom register for argument!"
) ? void (0) : __assert_fail ("Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 && \"Unexpected custom register for argument!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7428, __extension__ __PRETTY_FUNCTION__))
;
7429 CCValAssign &GPR1 = VA;
7430 SDValue MSWAsI64 = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgAsInt,
7431 DAG.getConstant(32, dl, MVT::i8));
7432 RegsToPass.push_back(std::make_pair(
7433 GPR1.getLocReg(), DAG.getZExtOrTrunc(MSWAsI64, dl, MVT::i32)));
7434
7435 if (I != E) {
7436 // If only 1 GPR was available, there will only be one custom GPR and
7437 // the argument will also pass in memory.
7438 CCValAssign &PeekArg = ArgLocs[I];
7439 if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) {
7440 assert(PeekArg.needsCustom() && "A second custom GPR is expected.")(static_cast <bool> (PeekArg.needsCustom() && "A second custom GPR is expected."
) ? void (0) : __assert_fail ("PeekArg.needsCustom() && \"A second custom GPR is expected.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7440, __extension__ __PRETTY_FUNCTION__))
;
7441 CCValAssign &GPR2 = ArgLocs[I++];
7442 RegsToPass.push_back(std::make_pair(
7443 GPR2.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, MVT::i32)));
7444 }
7445 }
7446 }
7447 }
7448
7449 if (!MemOpChains.empty())
7450 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
7451
7452 // For indirect calls, we need to save the TOC base to the stack for
7453 // restoration after the call.
7454 if (CFlags.IsIndirect) {
7455 assert(!CFlags.IsTailCall && "Indirect tail-calls not supported.")(static_cast <bool> (!CFlags.IsTailCall && "Indirect tail-calls not supported."
) ? void (0) : __assert_fail ("!CFlags.IsTailCall && \"Indirect tail-calls not supported.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7455, __extension__ __PRETTY_FUNCTION__))
;
7456 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister();
7457 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
7458 const MVT PtrVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
7459 const unsigned TOCSaveOffset =
7460 Subtarget.getFrameLowering()->getTOCSaveOffset();
7461
7462 setUsesTOCBasePtr(DAG);
7463 SDValue Val = DAG.getCopyFromReg(Chain, dl, TOCBaseReg, PtrVT);
7464 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
7465 SDValue StackPtr = DAG.getRegister(StackPtrReg, PtrVT);
7466 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
7467 Chain = DAG.getStore(
7468 Val.getValue(1), dl, Val, AddPtr,
7469 MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset));
7470 }
7471
7472 // Build a sequence of copy-to-reg nodes chained together with token chain
7473 // and flag operands which copy the outgoing args into the appropriate regs.
7474 SDValue InFlag;
7475 for (auto Reg : RegsToPass) {
7476 Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InFlag);
7477 InFlag = Chain.getValue(1);
7478 }
7479
7480 const int SPDiff = 0;
7481 return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart,
7482 Callee, SPDiff, NumBytes, Ins, InVals, CB);
7483}
7484
7485bool
7486PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
7487 MachineFunction &MF, bool isVarArg,
7488 const SmallVectorImpl<ISD::OutputArg> &Outs,
7489 LLVMContext &Context) const {
7490 SmallVector<CCValAssign, 16> RVLocs;
7491 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
7492 return CCInfo.CheckReturn(
7493 Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7494 ? RetCC_PPC_Cold
7495 : RetCC_PPC);
7496}
7497
7498SDValue
7499PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
7500 bool isVarArg,
7501 const SmallVectorImpl<ISD::OutputArg> &Outs,
7502 const SmallVectorImpl<SDValue> &OutVals,
7503 const SDLoc &dl, SelectionDAG &DAG) const {
7504 SmallVector<CCValAssign, 16> RVLocs;
7505 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
7506 *DAG.getContext());
7507 CCInfo.AnalyzeReturn(Outs,
7508 (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7509 ? RetCC_PPC_Cold
7510 : RetCC_PPC);
7511
7512 SDValue Flag;
7513 SmallVector<SDValue, 4> RetOps(1, Chain);
7514
7515 // Copy the result values into the output registers.
7516 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) {
7517 CCValAssign &VA = RVLocs[i];
7518 assert(VA.isRegLoc() && "Can only return in registers!")(static_cast <bool> (VA.isRegLoc() && "Can only return in registers!"
) ? void (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7518, __extension__ __PRETTY_FUNCTION__))
;
7519
7520 SDValue Arg = OutVals[RealResIdx];
7521
7522 switch (VA.getLocInfo()) {
7523 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7523)
;
7524 case CCValAssign::Full: break;
7525 case CCValAssign::AExt:
7526 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
7527 break;
7528 case CCValAssign::ZExt:
7529 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
7530 break;
7531 case CCValAssign::SExt:
7532 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
7533 break;
7534 }
7535 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
7536 bool isLittleEndian = Subtarget.isLittleEndian();
7537 // Legalize ret f64 -> ret 2 x i32.
7538 SDValue SVal =
7539 DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
7540 DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl));
7541 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag);
7542 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
7543 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
7544 DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl));
7545 Flag = Chain.getValue(1);
7546 VA = RVLocs[++i]; // skip ahead to next loc
7547 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag);
7548 } else
7549 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
7550 Flag = Chain.getValue(1);
7551 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
7552 }
7553
7554 RetOps[0] = Chain; // Update chain.
7555
7556 // Add the flag if we have it.
7557 if (Flag.getNode())
7558 RetOps.push_back(Flag);
7559
7560 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps);
7561}
7562
7563SDValue
7564PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
7565 SelectionDAG &DAG) const {
7566 SDLoc dl(Op);
7567
7568 // Get the correct type for integers.
7569 EVT IntVT = Op.getValueType();
7570
7571 // Get the inputs.
7572 SDValue Chain = Op.getOperand(0);
7573 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7574 // Build a DYNAREAOFFSET node.
7575 SDValue Ops[2] = {Chain, FPSIdx};
7576 SDVTList VTs = DAG.getVTList(IntVT);
7577 return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops);
7578}
7579
7580SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
7581 SelectionDAG &DAG) const {
7582 // When we pop the dynamic allocation we need to restore the SP link.
7583 SDLoc dl(Op);
7584
7585 // Get the correct type for pointers.
7586 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7587
7588 // Construct the stack pointer operand.
7589 bool isPPC64 = Subtarget.isPPC64();
7590 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
7591 SDValue StackPtr = DAG.getRegister(SP, PtrVT);
7592
7593 // Get the operands for the STACKRESTORE.
7594 SDValue Chain = Op.getOperand(0);
7595 SDValue SaveSP = Op.getOperand(1);
7596
7597 // Load the old link SP.
7598 SDValue LoadLinkSP =
7599 DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo());
7600
7601 // Restore the stack pointer.
7602 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP);
7603
7604 // Store the old link SP.
7605 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo());
7606}
7607
7608SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
7609 MachineFunction &MF = DAG.getMachineFunction();
7610 bool isPPC64 = Subtarget.isPPC64();
7611 EVT PtrVT = getPointerTy(MF.getDataLayout());
7612
7613 // Get current frame pointer save index. The users of this index will be
7614 // primarily DYNALLOC instructions.
7615 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
7616 int RASI = FI->getReturnAddrSaveIndex();
7617
7618 // If the frame pointer save index hasn't been defined yet.
7619 if (!RASI) {
7620 // Find out what the fix offset of the frame pointer save area.
7621 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
7622 // Allocate the frame index for frame pointer save area.
7623 RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false);
7624 // Save the result.
7625 FI->setReturnAddrSaveIndex(RASI);
7626 }
7627 return DAG.getFrameIndex(RASI, PtrVT);
7628}
7629
7630SDValue
7631PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
7632 MachineFunction &MF = DAG.getMachineFunction();
7633 bool isPPC64 = Subtarget.isPPC64();
7634 EVT PtrVT = getPointerTy(MF.getDataLayout());
7635
7636 // Get current frame pointer save index. The users of this index will be
7637 // primarily DYNALLOC instructions.
7638 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
7639 int FPSI = FI->getFramePointerSaveIndex();
7640
7641 // If the frame pointer save index hasn't been defined yet.
7642 if (!FPSI) {
7643 // Find out what the fix offset of the frame pointer save area.
7644 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
7645 // Allocate the frame index for frame pointer save area.
7646 FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true);
7647 // Save the result.
7648 FI->setFramePointerSaveIndex(FPSI);
7649 }
7650 return DAG.getFrameIndex(FPSI, PtrVT);
7651}
7652
7653SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
7654 SelectionDAG &DAG) const {
7655 MachineFunction &MF = DAG.getMachineFunction();
7656 // Get the inputs.
7657 SDValue Chain = Op.getOperand(0);
7658 SDValue Size = Op.getOperand(1);
7659 SDLoc dl(Op);
7660
7661 // Get the correct type for pointers.
7662 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7663 // Negate the size.
7664 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT,
7665 DAG.getConstant(0, dl, PtrVT), Size);
7666 // Construct a node for the frame pointer save index.
7667 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7668 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
7669 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other);
7670 if (hasInlineStackProbe(MF))
7671 return DAG.getNode(PPCISD::PROBED_ALLOCA, dl, VTs, Ops);
7672 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops);
7673}
7674
7675SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
7676 SelectionDAG &DAG) const {
7677 MachineFunction &MF = DAG.getMachineFunction();
7678
7679 bool isPPC64 = Subtarget.isPPC64();
7680 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7681
7682 int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false);
7683 return DAG.getFrameIndex(FI, PtrVT);
7684}
7685
7686SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
7687 SelectionDAG &DAG) const {
7688 SDLoc DL(Op);
7689 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL,
7690 DAG.getVTList(MVT::i32, MVT::Other),
7691 Op.getOperand(0), Op.getOperand(1));
7692}
7693
7694SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
7695 SelectionDAG &DAG) const {
7696 SDLoc DL(Op);
7697 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
7698 Op.getOperand(0), Op.getOperand(1));
7699}
7700
7701SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
7702 if (Op.getValueType().isVector())
7703 return LowerVectorLoad(Op, DAG);
7704
7705 assert(Op.getValueType() == MVT::i1 &&(static_cast <bool> (Op.getValueType() == MVT::i1 &&
"Custom lowering only for i1 loads") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i1 && \"Custom lowering only for i1 loads\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7706, __extension__ __PRETTY_FUNCTION__))
7706 "Custom lowering only for i1 loads")(static_cast <bool> (Op.getValueType() == MVT::i1 &&
"Custom lowering only for i1 loads") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i1 && \"Custom lowering only for i1 loads\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7706, __extension__ __PRETTY_FUNCTION__))
;
7707
7708 // First, load 8 bits into 32 bits, then truncate to 1 bit.
7709
7710 SDLoc dl(Op);
7711 LoadSDNode *LD = cast<LoadSDNode>(Op);
7712
7713 SDValue Chain = LD->getChain();
7714 SDValue BasePtr = LD->getBasePtr();
7715 MachineMemOperand *MMO = LD->getMemOperand();
7716
7717 SDValue NewLD =
7718 DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain,
7719 BasePtr, MVT::i8, MMO);
7720 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD);
7721
7722 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
7723 return DAG.getMergeValues(Ops, dl);
7724}
7725
7726SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
7727 if (Op.getOperand(1).getValueType().isVector())
7728 return LowerVectorStore(Op, DAG);
7729
7730 assert(Op.getOperand(1).getValueType() == MVT::i1 &&(static_cast <bool> (Op.getOperand(1).getValueType() ==
MVT::i1 && "Custom lowering only for i1 stores") ? void
(0) : __assert_fail ("Op.getOperand(1).getValueType() == MVT::i1 && \"Custom lowering only for i1 stores\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7731, __extension__ __PRETTY_FUNCTION__))
7731 "Custom lowering only for i1 stores")(static_cast <bool> (Op.getOperand(1).getValueType() ==
MVT::i1 && "Custom lowering only for i1 stores") ? void
(0) : __assert_fail ("Op.getOperand(1).getValueType() == MVT::i1 && \"Custom lowering only for i1 stores\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7731, __extension__ __PRETTY_FUNCTION__))
;
7732
7733 // First, zero extend to 32 bits, then use a truncating store to 8 bits.
7734
7735 SDLoc dl(Op);
7736 StoreSDNode *ST = cast<StoreSDNode>(Op);
7737
7738 SDValue Chain = ST->getChain();
7739 SDValue BasePtr = ST->getBasePtr();
7740 SDValue Value = ST->getValue();
7741 MachineMemOperand *MMO = ST->getMemOperand();
7742
7743 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()),
7744 Value);
7745 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO);
7746}
7747
7748// FIXME: Remove this once the ANDI glue bug is fixed:
7749SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
7750 assert(Op.getValueType() == MVT::i1 &&(static_cast <bool> (Op.getValueType() == MVT::i1 &&
"Custom lowering only for i1 results") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i1 && \"Custom lowering only for i1 results\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7751, __extension__ __PRETTY_FUNCTION__))
7751 "Custom lowering only for i1 results")(static_cast <bool> (Op.getValueType() == MVT::i1 &&
"Custom lowering only for i1 results") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i1 && \"Custom lowering only for i1 results\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7751, __extension__ __PRETTY_FUNCTION__))
;
7752
7753 SDLoc DL(Op);
7754 return DAG.getNode(PPCISD::ANDI_rec_1_GT_BIT, DL, MVT::i1, Op.getOperand(0));
7755}
7756
7757SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
7758 SelectionDAG &DAG) const {
7759
7760 // Implements a vector truncate that fits in a vector register as a shuffle.
7761 // We want to legalize vector truncates down to where the source fits in
7762 // a vector register (and target is therefore smaller than vector register
7763 // size). At that point legalization will try to custom lower the sub-legal
7764 // result and get here - where we can contain the truncate as a single target
7765 // operation.
7766
7767 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
7768 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
7769 //
7770 // We will implement it for big-endian ordering as this (where x denotes
7771 // undefined):
7772 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
7773 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
7774 //
7775 // The same operation in little-endian ordering will be:
7776 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
7777 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
7778
7779 EVT TrgVT = Op.getValueType();
7780 assert(TrgVT.isVector() && "Vector type expected.")(static_cast <bool> (TrgVT.isVector() && "Vector type expected."
) ? void (0) : __assert_fail ("TrgVT.isVector() && \"Vector type expected.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7780, __extension__ __PRETTY_FUNCTION__))
;
7781 unsigned TrgNumElts = TrgVT.getVectorNumElements();
7782 EVT EltVT = TrgVT.getVectorElementType();
7783 if (!isOperationCustom(Op.getOpcode(), TrgVT) ||
7784 TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(TrgNumElts) ||
7785 !isPowerOf2_32(EltVT.getSizeInBits()))
7786 return SDValue();
7787
7788 SDValue N1 = Op.getOperand(0);
7789 EVT SrcVT = N1.getValueType();
7790 unsigned SrcSize = SrcVT.getSizeInBits();
7791 if (SrcSize > 256 ||
7792 !isPowerOf2_32(SrcVT.getVectorNumElements()) ||
7793 !isPowerOf2_32(SrcVT.getVectorElementType().getSizeInBits()))
7794 return SDValue();
7795 if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2)
7796 return SDValue();
7797
7798 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
7799 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts);
7800
7801 SDLoc DL(Op);
7802 SDValue Op1, Op2;
7803 if (SrcSize == 256) {
7804 EVT VecIdxTy = getVectorIdxTy(DAG.getDataLayout());
7805 EVT SplitVT =
7806 N1.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
7807 unsigned SplitNumElts = SplitVT.getVectorNumElements();
7808 Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1,
7809 DAG.getConstant(0, DL, VecIdxTy));
7810 Op2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1,
7811 DAG.getConstant(SplitNumElts, DL, VecIdxTy));
7812 }
7813 else {
7814 Op1 = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL);
7815 Op2 = DAG.getUNDEF(WideVT);
7816 }
7817
7818 // First list the elements we want to keep.
7819 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
7820 SmallVector<int, 16> ShuffV;
7821 if (Subtarget.isLittleEndian())
7822 for (unsigned i = 0; i < TrgNumElts; ++i)
7823 ShuffV.push_back(i * SizeMult);
7824 else
7825 for (unsigned i = 1; i <= TrgNumElts; ++i)
7826 ShuffV.push_back(i * SizeMult - 1);
7827
7828 // Populate the remaining elements with undefs.
7829 for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
7830 // ShuffV.push_back(i + WideNumElts);
7831 ShuffV.push_back(WideNumElts + 1);
7832
7833 Op1 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op1);
7834 Op2 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op2);
7835 return DAG.getVectorShuffle(WideVT, DL, Op1, Op2, ShuffV);
7836}
7837
7838/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
7839/// possible.
7840SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
7841 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
7842 EVT ResVT = Op.getValueType();
7843 EVT CmpVT = Op.getOperand(0).getValueType();
7844 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
7845 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3);
7846 SDLoc dl(Op);
7847
7848 // Without power9-vector, we don't have native instruction for f128 comparison.
7849 // Following transformation to libcall is needed for setcc:
7850 // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE
7851 if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) {
7852 SDValue Z = DAG.getSetCC(
7853 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT),
7854 LHS, RHS, CC);
7855 SDValue Zero = DAG.getConstant(0, dl, Z.getValueType());
7856 return DAG.getSelectCC(dl, Z, Zero, TV, FV, ISD::SETNE);
7857 }
7858
7859 // Not FP, or using SPE? Not a fsel.
7860 if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() ||
7861 Subtarget.hasSPE())
7862 return Op;
7863
7864 SDNodeFlags Flags = Op.getNode()->getFlags();
7865
7866 // We have xsmaxcdp/xsmincdp which are OK to emit even in the
7867 // presence of infinities.
7868 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) {
7869 switch (CC) {
7870 default:
7871 break;
7872 case ISD::SETOGT:
7873 case ISD::SETGT:
7874 return DAG.getNode(PPCISD::XSMAXCDP, dl, Op.getValueType(), LHS, RHS);
7875 case ISD::SETOLT:
7876 case ISD::SETLT:
7877 return DAG.getNode(PPCISD::XSMINCDP, dl, Op.getValueType(), LHS, RHS);
7878 }
7879 }
7880
7881 // We might be able to do better than this under some circumstances, but in
7882 // general, fsel-based lowering of select is a finite-math-only optimization.
7883 // For more information, see section F.3 of the 2.06 ISA specification.
7884 // With ISA 3.0
7885 if ((!DAG.getTarget().Options.NoInfsFPMath && !Flags.hasNoInfs()) ||
7886 (!DAG.getTarget().Options.NoNaNsFPMath && !Flags.hasNoNaNs()))
7887 return Op;
7888
7889 // If the RHS of the comparison is a 0.0, we don't need to do the
7890 // subtraction at all.
7891 SDValue Sel1;
7892 if (isFloatingPointZero(RHS))
7893 switch (CC) {
7894 default: break; // SETUO etc aren't handled by fsel.
7895 case ISD::SETNE:
7896 std::swap(TV, FV);
7897 LLVM_FALLTHROUGH[[gnu::fallthrough]];
7898 case ISD::SETEQ:
7899 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
7900 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
7901 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV);
7902 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
7903 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1);
7904 return DAG.getNode(PPCISD::FSEL, dl, ResVT,
7905 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV);
7906 case ISD::SETULT:
7907 case ISD::SETLT:
7908 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt
7909 LLVM_FALLTHROUGH[[gnu::fallthrough]];
7910 case ISD::SETOGE:
7911 case ISD::SETGE:
7912 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
7913 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
7914 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV);
7915 case ISD::SETUGT:
7916 case ISD::SETGT:
7917 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt
7918 LLVM_FALLTHROUGH[[gnu::fallthrough]];
7919 case ISD::SETOLE:
7920 case ISD::SETLE:
7921 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
7922 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
7923 return DAG.getNode(PPCISD::FSEL, dl, ResVT,
7924 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV);
7925 }
7926
7927 SDValue Cmp;
7928 switch (CC) {
7929 default: break; // SETUO etc aren't handled by fsel.
7930 case ISD::SETNE:
7931 std::swap(TV, FV);
7932 LLVM_FALLTHROUGH[[gnu::fallthrough]];
7933 case ISD::SETEQ:
7934 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags);
7935 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
7936 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
7937 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
7938 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
7939 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1);
7940 return DAG.getNode(PPCISD::FSEL, dl, ResVT,
7941 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV);
7942 case ISD::SETULT:
7943 case ISD::SETLT:
7944 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags);
7945 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
7946 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
7947 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV);
7948 case ISD::SETOGE:
7949 case ISD::SETGE:
7950 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags);
7951 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
7952 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
7953 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
7954 case ISD::SETUGT:
7955 case ISD::SETGT:
7956 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags);
7957 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
7958 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
7959 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV);
7960 case ISD::SETOLE:
7961 case ISD::SETLE:
7962 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags);
7963 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
7964 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
7965 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
7966 }
7967 return Op;
7968}
7969
7970static unsigned getPPCStrictOpcode(unsigned Opc) {
7971 switch (Opc) {
7972 default:
7973 llvm_unreachable("No strict version of this opcode!")::llvm::llvm_unreachable_internal("No strict version of this opcode!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 7973)
;
7974 case PPCISD::FCTIDZ:
7975 return PPCISD::STRICT_FCTIDZ;
7976 case PPCISD::FCTIWZ:
7977 return PPCISD::STRICT_FCTIWZ;
7978 case PPCISD::FCTIDUZ:
7979 return PPCISD::STRICT_FCTIDUZ;
7980 case PPCISD::FCTIWUZ:
7981 return PPCISD::STRICT_FCTIWUZ;
7982 case PPCISD::FCFID:
7983 return PPCISD::STRICT_FCFID;
7984 case PPCISD::FCFIDU:
7985 return PPCISD::STRICT_FCFIDU;
7986 case PPCISD::FCFIDS:
7987 return PPCISD::STRICT_FCFIDS;
7988 case PPCISD::FCFIDUS:
7989 return PPCISD::STRICT_FCFIDUS;
7990 }
7991}
7992
7993static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
7994 const PPCSubtarget &Subtarget) {
7995 SDLoc dl(Op);
7996 bool IsStrict = Op->isStrictFPOpcode();
7997 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
7998 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
7999
8000 // TODO: Any other flags to propagate?
8001 SDNodeFlags Flags;
8002 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8003
8004 // For strict nodes, source is the second operand.
8005 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
8006 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
8007 assert(Src.getValueType().isFloatingPoint())(static_cast <bool> (Src.getValueType().isFloatingPoint
()) ? void (0) : __assert_fail ("Src.getValueType().isFloatingPoint()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8007, __extension__ __PRETTY_FUNCTION__))
;
8008 if (Src.getValueType() == MVT::f32) {
8009 if (IsStrict) {
8010 Src =
8011 DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
8012 DAG.getVTList(MVT::f64, MVT::Other), {Chain, Src}, Flags);
8013 Chain = Src.getValue(1);
8014 } else
8015 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
8016 }
8017 SDValue Conv;
8018 unsigned Opc = ISD::DELETED_NODE;
8019 switch (Op.getSimpleValueType().SimpleTy) {
8020 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!")::llvm::llvm_unreachable_internal("Unhandled FP_TO_INT type in custom expander!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8020)
;
8021 case MVT::i32:
8022 Opc = IsSigned ? PPCISD::FCTIWZ
8023 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ);
8024 break;
8025 case MVT::i64:
8026 assert((IsSigned || Subtarget.hasFPCVT()) &&(static_cast <bool> ((IsSigned || Subtarget.hasFPCVT())
&& "i64 FP_TO_UINT is supported only with FPCVT") ? void
(0) : __assert_fail ("(IsSigned || Subtarget.hasFPCVT()) && \"i64 FP_TO_UINT is supported only with FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8027, __extension__ __PRETTY_FUNCTION__))
8027 "i64 FP_TO_UINT is supported only with FPCVT")(static_cast <bool> ((IsSigned || Subtarget.hasFPCVT())
&& "i64 FP_TO_UINT is supported only with FPCVT") ? void
(0) : __assert_fail ("(IsSigned || Subtarget.hasFPCVT()) && \"i64 FP_TO_UINT is supported only with FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8027, __extension__ __PRETTY_FUNCTION__))
;
8028 Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ;
8029 }
8030 if (IsStrict) {
8031 Opc = getPPCStrictOpcode(Opc);
8032 Conv = DAG.getNode(Opc, dl, DAG.getVTList(MVT::f64, MVT::Other),
8033 {Chain, Src}, Flags);
8034 } else {
8035 Conv = DAG.getNode(Opc, dl, MVT::f64, Src);
8036 }
8037 return Conv;
8038}
8039
8040void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
8041 SelectionDAG &DAG,
8042 const SDLoc &dl) const {
8043 SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
8044 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8045 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8046 bool IsStrict = Op->isStrictFPOpcode();
8047
8048 // Convert the FP value to an int value through memory.
8049 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
8050 (IsSigned || Subtarget.hasFPCVT());
8051 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64);
8052 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex();
8053 MachinePointerInfo MPI =
8054 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
8055
8056 // Emit a store to the stack slot.
8057 SDValue Chain = IsStrict ? Tmp.getValue(1) : DAG.getEntryNode();
8058 Align Alignment(DAG.getEVTAlign(Tmp.getValueType()));
8059 if (i32Stack) {
8060 MachineFunction &MF = DAG.getMachineFunction();
8061 Alignment = Align(4);
8062 MachineMemOperand *MMO =
8063 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Alignment);
8064 SDValue Ops[] = { Chain, Tmp, FIPtr };
8065 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl,
8066 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO);
8067 } else
8068 Chain = DAG.getStore(Chain, dl, Tmp, FIPtr, MPI, Alignment);
8069
8070 // Result is a load from the stack slot. If loading 4 bytes, make sure to
8071 // add in a bias on big endian.
8072 if (Op.getValueType() == MVT::i32 && !i32Stack) {
8073 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr,
8074 DAG.getConstant(4, dl, FIPtr.getValueType()));
8075 MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4);
8076 }
8077
8078 RLI.Chain = Chain;
8079 RLI.Ptr = FIPtr;
8080 RLI.MPI = MPI;
8081 RLI.Alignment = Alignment;
8082}
8083
8084/// Custom lowers floating point to integer conversions to use
8085/// the direct move instructions available in ISA 2.07 to avoid the
8086/// need for load/store combinations.
8087SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
8088 SelectionDAG &DAG,
8089 const SDLoc &dl) const {
8090 SDValue Conv = convertFPToInt(Op, DAG, Subtarget);
8091 SDValue Mov = DAG.getNode(PPCISD::MFVSR, dl, Op.getValueType(), Conv);
8092 if (Op->isStrictFPOpcode())
8093 return DAG.getMergeValues({Mov, Conv.getValue(1)}, dl);
8094 else
8095 return Mov;
8096}
8097
8098SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
8099 const SDLoc &dl) const {
8100 bool IsStrict = Op->isStrictFPOpcode();
8101 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8102 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8103 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
8104 EVT SrcVT = Src.getValueType();
8105 EVT DstVT = Op.getValueType();
8106
8107 // FP to INT conversions are legal for f128.
8108 if (SrcVT == MVT::f128)
8109 return Subtarget.hasP9Vector() ? Op : SDValue();
8110
8111 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
8112 // PPC (the libcall is not available).
8113 if (SrcVT == MVT::ppcf128) {
8114 if (DstVT == MVT::i32) {
8115 // TODO: Conservatively pass only nofpexcept flag here. Need to check and
8116 // set other fast-math flags to FP operations in both strict and
8117 // non-strict cases. (FP_TO_SINT, FSUB)
8118 SDNodeFlags Flags;
8119 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8120
8121 if (IsSigned) {
8122 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src,
8123 DAG.getIntPtrConstant(0, dl));
8124 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src,
8125 DAG.getIntPtrConstant(1, dl));
8126
8127 // Add the two halves of the long double in round-to-zero mode, and use
8128 // a smaller FP_TO_SINT.
8129 if (IsStrict) {
8130 SDValue Res = DAG.getNode(PPCISD::STRICT_FADDRTZ, dl,
8131 DAG.getVTList(MVT::f64, MVT::Other),
8132 {Op.getOperand(0), Lo, Hi}, Flags);
8133 return DAG.getNode(ISD::STRICT_FP_TO_SINT, dl,
8134 DAG.getVTList(MVT::i32, MVT::Other),
8135 {Res.getValue(1), Res}, Flags);
8136 } else {
8137 SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi);
8138 return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res);
8139 }
8140 } else {
8141 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
8142 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31));
8143 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
8144 SDValue SignMask = DAG.getConstant(0x80000000, dl, DstVT);
8145 if (IsStrict) {
8146 // Sel = Src < 0x80000000
8147 // FltOfs = select Sel, 0.0, 0x80000000
8148 // IntOfs = select Sel, 0, 0x80000000
8149 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
8150 SDValue Chain = Op.getOperand(0);
8151 EVT SetCCVT =
8152 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
8153 EVT DstSetCCVT =
8154 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
8155 SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
8156 Chain, true);
8157 Chain = Sel.getValue(1);
8158
8159 SDValue FltOfs = DAG.getSelect(
8160 dl, SrcVT, Sel, DAG.getConstantFP(0.0, dl, SrcVT), Cst);
8161 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
8162
8163 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl,
8164 DAG.getVTList(SrcVT, MVT::Other),
8165 {Chain, Src, FltOfs}, Flags);
8166 Chain = Val.getValue(1);
8167 SDValue SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl,
8168 DAG.getVTList(DstVT, MVT::Other),
8169 {Chain, Val}, Flags);
8170 Chain = SInt.getValue(1);
8171 SDValue IntOfs = DAG.getSelect(
8172 dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT), SignMask);
8173 SDValue Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
8174 return DAG.getMergeValues({Result, Chain}, dl);
8175 } else {
8176 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
8177 // FIXME: generated code sucks.
8178 SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, Src, Cst);
8179 True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True);
8180 True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, SignMask);
8181 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
8182 return DAG.getSelectCC(dl, Src, Cst, True, False, ISD::SETGE);
8183 }
8184 }
8185 }
8186
8187 return SDValue();
8188 }
8189
8190 if (Subtarget.hasDirectMove() && Subtarget.isPPC64())
8191 return LowerFP_TO_INTDirectMove(Op, DAG, dl);
8192
8193 ReuseLoadInfo RLI;
8194 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8195
8196 return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI,
8197 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges);
8198}
8199
8200// We're trying to insert a regular store, S, and then a load, L. If the
8201// incoming value, O, is a load, we might just be able to have our load use the
8202// address used by O. However, we don't know if anything else will store to
8203// that address before we can load from it. To prevent this situation, we need
8204// to insert our load, L, into the chain as a peer of O. To do this, we give L
8205// the same chain operand as O, we create a token factor from the chain results
8206// of O and L, and we replace all uses of O's chain result with that token
8207// factor (see spliceIntoChain below for this last part).
8208bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
8209 ReuseLoadInfo &RLI,
8210 SelectionDAG &DAG,
8211 ISD::LoadExtType ET) const {
8212 // Conservatively skip reusing for constrained FP nodes.
8213 if (Op->isStrictFPOpcode())
8214 return false;
8215
8216 SDLoc dl(Op);
8217 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT &&
8218 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32);
8219 if (ET == ISD::NON_EXTLOAD &&
8220 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) &&
8221 isOperationLegalOrCustom(Op.getOpcode(),
8222 Op.getOperand(0).getValueType())) {
8223
8224 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8225 return true;
8226 }
8227
8228 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op);
8229 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() ||
8230 LD->isNonTemporal())
8231 return false;
8232 if (LD->getMemoryVT() != MemVT)
8233 return false;
8234
8235 // If the result of the load is an illegal type, then we can't build a
8236 // valid chain for reuse since the legalised loads and token factor node that
8237 // ties the legalised loads together uses a different output chain then the
8238 // illegal load.
8239 if (!isTypeLegal(LD->getValueType(0)))
8240 return false;
8241
8242 RLI.Ptr = LD->getBasePtr();
8243 if (LD->isIndexed() && !LD->getOffset().isUndef()) {
8244 assert(LD->getAddressingMode() == ISD::PRE_INC &&(static_cast <bool> (LD->getAddressingMode() == ISD::
PRE_INC && "Non-pre-inc AM on PPC?") ? void (0) : __assert_fail
("LD->getAddressingMode() == ISD::PRE_INC && \"Non-pre-inc AM on PPC?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8245, __extension__ __PRETTY_FUNCTION__))
8245 "Non-pre-inc AM on PPC?")(static_cast <bool> (LD->getAddressingMode() == ISD::
PRE_INC && "Non-pre-inc AM on PPC?") ? void (0) : __assert_fail
("LD->getAddressingMode() == ISD::PRE_INC && \"Non-pre-inc AM on PPC?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8245, __extension__ __PRETTY_FUNCTION__))
;
8246 RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr,
8247 LD->getOffset());
8248 }
8249
8250 RLI.Chain = LD->getChain();
8251 RLI.MPI = LD->getPointerInfo();
8252 RLI.IsDereferenceable = LD->isDereferenceable();
8253 RLI.IsInvariant = LD->isInvariant();
8254 RLI.Alignment = LD->getAlign();
8255 RLI.AAInfo = LD->getAAInfo();
8256 RLI.Ranges = LD->getRanges();
8257
8258 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1);
8259 return true;
8260}
8261
8262// Given the head of the old chain, ResChain, insert a token factor containing
8263// it and NewResChain, and make users of ResChain now be users of that token
8264// factor.
8265// TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead.
8266void PPCTargetLowering::spliceIntoChain(SDValue ResChain,
8267 SDValue NewResChain,
8268 SelectionDAG &DAG) const {
8269 if (!ResChain)
8270 return;
8271
8272 SDLoc dl(NewResChain);
8273
8274 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
8275 NewResChain, DAG.getUNDEF(MVT::Other));
8276 assert(TF.getNode() != NewResChain.getNode() &&(static_cast <bool> (TF.getNode() != NewResChain.getNode
() && "A new TF really is required here") ? void (0) :
__assert_fail ("TF.getNode() != NewResChain.getNode() && \"A new TF really is required here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8277, __extension__ __PRETTY_FUNCTION__))
8277 "A new TF really is required here")(static_cast <bool> (TF.getNode() != NewResChain.getNode
() && "A new TF really is required here") ? void (0) :
__assert_fail ("TF.getNode() != NewResChain.getNode() && \"A new TF really is required here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8277, __extension__ __PRETTY_FUNCTION__))
;
8278
8279 DAG.ReplaceAllUsesOfValueWith(ResChain, TF);
8280 DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain);
8281}
8282
8283/// Analyze profitability of direct move
8284/// prefer float load to int load plus direct move
8285/// when there is no integer use of int load
8286bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
8287 SDNode *Origin = Op.getOperand(0).getNode();
8288 if (Origin->getOpcode() != ISD::LOAD)
8289 return true;
8290
8291 // If there is no LXSIBZX/LXSIHZX, like Power8,
8292 // prefer direct move if the memory size is 1 or 2 bytes.
8293 MachineMemOperand *MMO = cast<LoadSDNode>(Origin)->getMemOperand();
8294 if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2)
8295 return true;
8296
8297 for (SDNode::use_iterator UI = Origin->use_begin(),
8298 UE = Origin->use_end();
8299 UI != UE; ++UI) {
8300
8301 // Only look at the users of the loaded value.
8302 if (UI.getUse().get().getResNo() != 0)
8303 continue;
8304
8305 if (UI->getOpcode() != ISD::SINT_TO_FP &&
8306 UI->getOpcode() != ISD::UINT_TO_FP &&
8307 UI->getOpcode() != ISD::STRICT_SINT_TO_FP &&
8308 UI->getOpcode() != ISD::STRICT_UINT_TO_FP)
8309 return true;
8310 }
8311
8312 return false;
8313}
8314
8315static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG,
8316 const PPCSubtarget &Subtarget,
8317 SDValue Chain = SDValue()) {
8318 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8319 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8320 SDLoc dl(Op);
8321
8322 // TODO: Any other flags to propagate?
8323 SDNodeFlags Flags;
8324 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8325
8326 // If we have FCFIDS, then use it when converting to single-precision.
8327 // Otherwise, convert to double-precision and then round.
8328 bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT();
8329 unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS)
8330 : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU);
8331 EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64;
8332 if (Op->isStrictFPOpcode()) {
8333 if (!Chain)
8334 Chain = Op.getOperand(0);
8335 return DAG.getNode(getPPCStrictOpcode(ConvOpc), dl,
8336 DAG.getVTList(ConvTy, MVT::Other), {Chain, Src}, Flags);
8337 } else
8338 return DAG.getNode(ConvOpc, dl, ConvTy, Src);
8339}
8340
8341/// Custom lowers integer to floating point conversions to use
8342/// the direct move instructions available in ISA 2.07 to avoid the
8343/// need for load/store combinations.
8344SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
8345 SelectionDAG &DAG,
8346 const SDLoc &dl) const {
8347 assert((Op.getValueType() == MVT::f32 ||(static_cast <bool> ((Op.getValueType() == MVT::f32 || Op
.getValueType() == MVT::f64) && "Invalid floating point type as target of conversion"
) ? void (0) : __assert_fail ("(Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && \"Invalid floating point type as target of conversion\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8349, __extension__ __PRETTY_FUNCTION__))
8348 Op.getValueType() == MVT::f64) &&(static_cast <bool> ((Op.getValueType() == MVT::f32 || Op
.getValueType() == MVT::f64) && "Invalid floating point type as target of conversion"
) ? void (0) : __assert_fail ("(Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && \"Invalid floating point type as target of conversion\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8349, __extension__ __PRETTY_FUNCTION__))
8349 "Invalid floating point type as target of conversion")(static_cast <bool> ((Op.getValueType() == MVT::f32 || Op
.getValueType() == MVT::f64) && "Invalid floating point type as target of conversion"
) ? void (0) : __assert_fail ("(Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && \"Invalid floating point type as target of conversion\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8349, __extension__ __PRETTY_FUNCTION__))
;
8350 assert(Subtarget.hasFPCVT() &&(static_cast <bool> (Subtarget.hasFPCVT() && "Int to FP conversions with direct moves require FPCVT"
) ? void (0) : __assert_fail ("Subtarget.hasFPCVT() && \"Int to FP conversions with direct moves require FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8351, __extension__ __PRETTY_FUNCTION__))
8351 "Int to FP conversions with direct moves require FPCVT")(static_cast <bool> (Subtarget.hasFPCVT() && "Int to FP conversions with direct moves require FPCVT"
) ? void (0) : __assert_fail ("Subtarget.hasFPCVT() && \"Int to FP conversions with direct moves require FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8351, __extension__ __PRETTY_FUNCTION__))
;
8352 SDValue Src = Op.getOperand(Op->isStrictFPOpcode() ? 1 : 0);
8353 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
8354 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP ||
8355 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8356 unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA;
8357 SDValue Mov = DAG.getNode(MovOpc, dl, MVT::f64, Src);
8358 return convertIntToFP(Op, Mov, DAG, Subtarget);
8359}
8360
8361static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
8362
8363 EVT VecVT = Vec.getValueType();
8364 assert(VecVT.isVector() && "Expected a vector type.")(static_cast <bool> (VecVT.isVector() && "Expected a vector type."
) ? void (0) : __assert_fail ("VecVT.isVector() && \"Expected a vector type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8364, __extension__ __PRETTY_FUNCTION__))
;
8365 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width.")(static_cast <bool> (VecVT.getSizeInBits() < 128 &&
"Vector is already full width.") ? void (0) : __assert_fail (
"VecVT.getSizeInBits() < 128 && \"Vector is already full width.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8365, __extension__ __PRETTY_FUNCTION__))
;
8366
8367 EVT EltVT = VecVT.getVectorElementType();
8368 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8369 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts);
8370
8371 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements();
8372 SmallVector<SDValue, 16> Ops(NumConcat);
8373 Ops[0] = Vec;
8374 SDValue UndefVec = DAG.getUNDEF(VecVT);
8375 for (unsigned i = 1; i < NumConcat; ++i)
8376 Ops[i] = UndefVec;
8377
8378 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops);
8379}
8380
8381SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
8382 const SDLoc &dl) const {
8383 bool IsStrict = Op->isStrictFPOpcode();
8384 unsigned Opc = Op.getOpcode();
8385 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
8386 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP ||(static_cast <bool> ((Opc == ISD::UINT_TO_FP || Opc == ISD
::SINT_TO_FP || Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::
STRICT_SINT_TO_FP) && "Unexpected conversion type") ?
void (0) : __assert_fail ("(Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) && \"Unexpected conversion type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8388, __extension__ __PRETTY_FUNCTION__))
8387 Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) &&(static_cast <bool> ((Opc == ISD::UINT_TO_FP || Opc == ISD
::SINT_TO_FP || Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::
STRICT_SINT_TO_FP) && "Unexpected conversion type") ?
void (0) : __assert_fail ("(Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) && \"Unexpected conversion type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8388, __extension__ __PRETTY_FUNCTION__))
8388 "Unexpected conversion type")(static_cast <bool> ((Opc == ISD::UINT_TO_FP || Opc == ISD
::SINT_TO_FP || Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::
STRICT_SINT_TO_FP) && "Unexpected conversion type") ?
void (0) : __assert_fail ("(Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) && \"Unexpected conversion type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8388, __extension__ __PRETTY_FUNCTION__))
;
8389 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) &&(static_cast <bool> ((Op.getValueType() == MVT::v2f64 ||
Op.getValueType() == MVT::v4f32) && "Supports conversions to v2f64/v4f32 only."
) ? void (0) : __assert_fail ("(Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && \"Supports conversions to v2f64/v4f32 only.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8390, __extension__ __PRETTY_FUNCTION__))
8390 "Supports conversions to v2f64/v4f32 only.")(static_cast <bool> ((Op.getValueType() == MVT::v2f64 ||
Op.getValueType() == MVT::v4f32) && "Supports conversions to v2f64/v4f32 only."
) ? void (0) : __assert_fail ("(Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && \"Supports conversions to v2f64/v4f32 only.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8390, __extension__ __PRETTY_FUNCTION__))
;
8391
8392 // TODO: Any other flags to propagate?
8393 SDNodeFlags Flags;
8394 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8395
8396 bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
8397 bool FourEltRes = Op.getValueType() == MVT::v4f32;
8398
8399 SDValue Wide = widenVec(DAG, Src, dl);
8400 EVT WideVT = Wide.getValueType();
8401 unsigned WideNumElts = WideVT.getVectorNumElements();
8402 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64;
8403
8404 SmallVector<int, 16> ShuffV;
8405 for (unsigned i = 0; i < WideNumElts; ++i)
8406 ShuffV.push_back(i + WideNumElts);
8407
8408 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2;
8409 int SaveElts = FourEltRes ? 4 : 2;
8410 if (Subtarget.isLittleEndian())
8411 for (int i = 0; i < SaveElts; i++)
8412 ShuffV[i * Stride] = i;
8413 else
8414 for (int i = 1; i <= SaveElts; i++)
8415 ShuffV[i * Stride - 1] = i - 1;
8416
8417 SDValue ShuffleSrc2 =
8418 SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT);
8419 SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV);
8420
8421 SDValue Extend;
8422 if (SignedConv) {
8423 Arrange = DAG.getBitcast(IntermediateVT, Arrange);
8424 EVT ExtVT = Src.getValueType();
8425 if (Subtarget.hasP9Altivec())
8426 ExtVT = EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(),
8427 IntermediateVT.getVectorNumElements());
8428
8429 Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange,
8430 DAG.getValueType(ExtVT));
8431 } else
8432 Extend = DAG.getNode(ISD::BITCAST, dl, IntermediateVT, Arrange);
8433
8434 if (IsStrict)
8435 return DAG.getNode(Opc, dl, DAG.getVTList(Op.getValueType(), MVT::Other),
8436 {Op.getOperand(0), Extend}, Flags);
8437
8438 return DAG.getNode(Opc, dl, Op.getValueType(), Extend);
8439}
8440
8441SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
8442 SelectionDAG &DAG) const {
8443 SDLoc dl(Op);
8444 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8445 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8446 bool IsStrict = Op->isStrictFPOpcode();
8447 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
8448 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
8449
8450 // TODO: Any other flags to propagate?
8451 SDNodeFlags Flags;
8452 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8453
8454 EVT InVT = Src.getValueType();
8455 EVT OutVT = Op.getValueType();
8456 if (OutVT.isVector() && OutVT.isFloatingPoint() &&
8457 isOperationCustom(Op.getOpcode(), InVT))
8458 return LowerINT_TO_FPVector(Op, DAG, dl);
8459
8460 // Conversions to f128 are legal.
8461 if (Op.getValueType() == MVT::f128)
8462 return Subtarget.hasP9Vector() ? Op : SDValue();
8463
8464 // Don't handle ppc_fp128 here; let it be lowered to a libcall.
8465 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
8466 return SDValue();
8467
8468 if (Src.getValueType() == MVT::i1) {
8469 SDValue Sel = DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Src,
8470 DAG.getConstantFP(1.0, dl, Op.getValueType()),
8471 DAG.getConstantFP(0.0, dl, Op.getValueType()));
8472 if (IsStrict)
8473 return DAG.getMergeValues({Sel, Chain}, dl);
8474 else
8475 return Sel;
8476 }
8477
8478 // If we have direct moves, we can do all the conversion, skip the store/load
8479 // however, without FPCVT we can't do most conversions.
8480 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) &&
8481 Subtarget.isPPC64() && Subtarget.hasFPCVT())
8482 return LowerINT_TO_FPDirectMove(Op, DAG, dl);
8483
8484 assert((IsSigned || Subtarget.hasFPCVT()) &&(static_cast <bool> ((IsSigned || Subtarget.hasFPCVT())
&& "UINT_TO_FP is supported only with FPCVT") ? void
(0) : __assert_fail ("(IsSigned || Subtarget.hasFPCVT()) && \"UINT_TO_FP is supported only with FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8485, __extension__ __PRETTY_FUNCTION__))
8485 "UINT_TO_FP is supported only with FPCVT")(static_cast <bool> ((IsSigned || Subtarget.hasFPCVT())
&& "UINT_TO_FP is supported only with FPCVT") ? void
(0) : __assert_fail ("(IsSigned || Subtarget.hasFPCVT()) && \"UINT_TO_FP is supported only with FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8485, __extension__ __PRETTY_FUNCTION__))
;
8486
8487 if (Src.getValueType() == MVT::i64) {
8488 SDValue SINT = Src;
8489 // When converting to single-precision, we actually need to convert
8490 // to double-precision first and then round to single-precision.
8491 // To avoid double-rounding effects during that operation, we have
8492 // to prepare the input operand. Bits that might be truncated when
8493 // converting to double-precision are replaced by a bit that won't
8494 // be lost at this stage, but is below the single-precision rounding
8495 // position.
8496 //
8497 // However, if -enable-unsafe-fp-math is in effect, accept double
8498 // rounding to avoid the extra overhead.
8499 if (Op.getValueType() == MVT::f32 &&
8500 !Subtarget.hasFPCVT() &&
8501 !DAG.getTarget().Options.UnsafeFPMath) {
8502
8503 // Twiddle input to make sure the low 11 bits are zero. (If this
8504 // is the case, we are guaranteed the value will fit into the 53 bit
8505 // mantissa of an IEEE double-precision value without rounding.)
8506 // If any of those low 11 bits were not zero originally, make sure
8507 // bit 12 (value 2048) is set instead, so that the final rounding
8508 // to single-precision gets the correct result.
8509 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64,
8510 SINT, DAG.getConstant(2047, dl, MVT::i64));
8511 Round = DAG.getNode(ISD::ADD, dl, MVT::i64,
8512 Round, DAG.getConstant(2047, dl, MVT::i64));
8513 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT);
8514 Round = DAG.getNode(ISD::AND, dl, MVT::i64,
8515 Round, DAG.getConstant(-2048, dl, MVT::i64));
8516
8517 // However, we cannot use that value unconditionally: if the magnitude
8518 // of the input value is small, the bit-twiddling we did above might
8519 // end up visibly changing the output. Fortunately, in that case, we
8520 // don't need to twiddle bits since the original input will convert
8521 // exactly to double-precision floating-point already. Therefore,
8522 // construct a conditional to use the original value if the top 11
8523 // bits are all sign-bit copies, and use the rounded value computed
8524 // above otherwise.
8525 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64,
8526 SINT, DAG.getConstant(53, dl, MVT::i32));
8527 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64,
8528 Cond, DAG.getConstant(1, dl, MVT::i64));
8529 Cond = DAG.getSetCC(
8530 dl,
8531 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
8532 Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT);
8533
8534 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT);
8535 }
8536
8537 ReuseLoadInfo RLI;
8538 SDValue Bits;
8539
8540 MachineFunction &MF = DAG.getMachineFunction();
8541 if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) {
8542 Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI,
8543 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges);
8544 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
8545 } else if (Subtarget.hasLFIWAX() &&
8546 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
8547 MachineMemOperand *MMO =
8548 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
8549 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
8550 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8551 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
8552 DAG.getVTList(MVT::f64, MVT::Other),
8553 Ops, MVT::i32, MMO);
8554 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
8555 } else if (Subtarget.hasFPCVT() &&
8556 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
8557 MachineMemOperand *MMO =
8558 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
8559 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
8560 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8561 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
8562 DAG.getVTList(MVT::f64, MVT::Other),
8563 Ops, MVT::i32, MMO);
8564 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
8565 } else if (((Subtarget.hasLFIWAX() &&
8566 SINT.getOpcode() == ISD::SIGN_EXTEND) ||
8567 (Subtarget.hasFPCVT() &&
8568 SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
8569 SINT.getOperand(0).getValueType() == MVT::i32) {
8570 MachineFrameInfo &MFI = MF.getFrameInfo();
8571 EVT PtrVT = getPointerTy(DAG.getDataLayout());
8572
8573 int FrameIdx = MFI.CreateStackObject(4, Align(4), false);
8574 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
8575
8576 SDValue Store = DAG.getStore(Chain, dl, SINT.getOperand(0), FIdx,
8577 MachinePointerInfo::getFixedStack(
8578 DAG.getMachineFunction(), FrameIdx));
8579 Chain = Store;
8580
8581 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&(static_cast <bool> (cast<StoreSDNode>(Store)->
getMemoryVT() == MVT::i32 && "Expected an i32 store")
? void (0) : __assert_fail ("cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && \"Expected an i32 store\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8582, __extension__ __PRETTY_FUNCTION__))
8582 "Expected an i32 store")(static_cast <bool> (cast<StoreSDNode>(Store)->
getMemoryVT() == MVT::i32 && "Expected an i32 store")
? void (0) : __assert_fail ("cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && \"Expected an i32 store\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8582, __extension__ __PRETTY_FUNCTION__))
;
8583
8584 RLI.Ptr = FIdx;
8585 RLI.Chain = Chain;
8586 RLI.MPI =
8587 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
8588 RLI.Alignment = Align(4);
8589
8590 MachineMemOperand *MMO =
8591 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
8592 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
8593 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8594 Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ?
8595 PPCISD::LFIWZX : PPCISD::LFIWAX,
8596 dl, DAG.getVTList(MVT::f64, MVT::Other),
8597 Ops, MVT::i32, MMO);
8598 Chain = Bits.getValue(1);
8599 } else
8600 Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT);
8601
8602 SDValue FP = convertIntToFP(Op, Bits, DAG, Subtarget, Chain);
8603 if (IsStrict)
8604 Chain = FP.getValue(1);
8605
8606 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
8607 if (IsStrict)
8608 FP = DAG.getNode(ISD::STRICT_FP_ROUND, dl,
8609 DAG.getVTList(MVT::f32, MVT::Other),
8610 {Chain, FP, DAG.getIntPtrConstant(0, dl)}, Flags);
8611 else
8612 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP,
8613 DAG.getIntPtrConstant(0, dl));
8614 }
8615 return FP;
8616 }
8617
8618 assert(Src.getValueType() == MVT::i32 &&(static_cast <bool> (Src.getValueType() == MVT::i32 &&
"Unhandled INT_TO_FP type in custom expander!") ? void (0) :
__assert_fail ("Src.getValueType() == MVT::i32 && \"Unhandled INT_TO_FP type in custom expander!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8619, __extension__ __PRETTY_FUNCTION__))
8619 "Unhandled INT_TO_FP type in custom expander!")(static_cast <bool> (Src.getValueType() == MVT::i32 &&
"Unhandled INT_TO_FP type in custom expander!") ? void (0) :
__assert_fail ("Src.getValueType() == MVT::i32 && \"Unhandled INT_TO_FP type in custom expander!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8619, __extension__ __PRETTY_FUNCTION__))
;
8620 // Since we only generate this in 64-bit mode, we can take advantage of
8621 // 64-bit registers. In particular, sign extend the input value into the
8622 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
8623 // then lfd it and fcfid it.
8624 MachineFunction &MF = DAG.getMachineFunction();
8625 MachineFrameInfo &MFI = MF.getFrameInfo();
8626 EVT PtrVT = getPointerTy(MF.getDataLayout());
8627
8628 SDValue Ld;
8629 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
8630 ReuseLoadInfo RLI;
8631 bool ReusingLoad;
8632 if (!(ReusingLoad = canReuseLoadAddress(Src, MVT::i32, RLI, DAG))) {
8633 int FrameIdx = MFI.CreateStackObject(4, Align(4), false);
8634 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
8635
8636 SDValue Store = DAG.getStore(Chain, dl, Src, FIdx,
8637 MachinePointerInfo::getFixedStack(
8638 DAG.getMachineFunction(), FrameIdx));
8639 Chain = Store;
8640
8641 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&(static_cast <bool> (cast<StoreSDNode>(Store)->
getMemoryVT() == MVT::i32 && "Expected an i32 store")
? void (0) : __assert_fail ("cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && \"Expected an i32 store\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8642, __extension__ __PRETTY_FUNCTION__))
8642 "Expected an i32 store")(static_cast <bool> (cast<StoreSDNode>(Store)->
getMemoryVT() == MVT::i32 && "Expected an i32 store")
? void (0) : __assert_fail ("cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && \"Expected an i32 store\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8642, __extension__ __PRETTY_FUNCTION__))
;
8643
8644 RLI.Ptr = FIdx;
8645 RLI.Chain = Chain;
8646 RLI.MPI =
8647 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
8648 RLI.Alignment = Align(4);
8649 }
8650
8651 MachineMemOperand *MMO =
8652 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
8653 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
8654 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8655 Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
8656 DAG.getVTList(MVT::f64, MVT::Other), Ops,
8657 MVT::i32, MMO);
8658 Chain = Ld.getValue(1);
8659 if (ReusingLoad)
8660 spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG);
8661 } else {
8662 assert(Subtarget.isPPC64() &&(static_cast <bool> (Subtarget.isPPC64() && "i32->FP without LFIWAX supported only on PPC64"
) ? void (0) : __assert_fail ("Subtarget.isPPC64() && \"i32->FP without LFIWAX supported only on PPC64\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8663, __extension__ __PRETTY_FUNCTION__))
8663 "i32->FP without LFIWAX supported only on PPC64")(static_cast <bool> (Subtarget.isPPC64() && "i32->FP without LFIWAX supported only on PPC64"
) ? void (0) : __assert_fail ("Subtarget.isPPC64() && \"i32->FP without LFIWAX supported only on PPC64\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8663, __extension__ __PRETTY_FUNCTION__))
;
8664
8665 int FrameIdx = MFI.CreateStackObject(8, Align(8), false);
8666 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
8667
8668 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, Src);
8669
8670 // STD the extended value into the stack slot.
8671 SDValue Store = DAG.getStore(
8672 Chain, dl, Ext64, FIdx,
8673 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx));
8674 Chain = Store;
8675
8676 // Load the value as a double.
8677 Ld = DAG.getLoad(
8678 MVT::f64, dl, Chain, FIdx,
8679 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx));
8680 Chain = Ld.getValue(1);
8681 }
8682
8683 // FCFID it and return it.
8684 SDValue FP = convertIntToFP(Op, Ld, DAG, Subtarget, Chain);
8685 if (IsStrict)
8686 Chain = FP.getValue(1);
8687 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
8688 if (IsStrict)
8689 FP = DAG.getNode(ISD::STRICT_FP_ROUND, dl,
8690 DAG.getVTList(MVT::f32, MVT::Other),
8691 {Chain, FP, DAG.getIntPtrConstant(0, dl)}, Flags);
8692 else
8693 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP,
8694 DAG.getIntPtrConstant(0, dl));
8695 }
8696 return FP;
8697}
8698
8699SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
8700 SelectionDAG &DAG) const {
8701 SDLoc dl(Op);
8702 /*
8703 The rounding mode is in bits 30:31 of FPSR, and has the following
8704 settings:
8705 00 Round to nearest
8706 01 Round to 0
8707 10 Round to +inf
8708 11 Round to -inf
8709
8710 FLT_ROUNDS, on the other hand, expects the following:
8711 -1 Undefined
8712 0 Round to 0
8713 1 Round to nearest
8714 2 Round to +inf
8715 3 Round to -inf
8716
8717 To perform the conversion, we do:
8718 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1))
8719 */
8720
8721 MachineFunction &MF = DAG.getMachineFunction();
8722 EVT VT = Op.getValueType();
8723 EVT PtrVT = getPointerTy(MF.getDataLayout());
8724
8725 // Save FP Control Word to register
8726 SDValue Chain = Op.getOperand(0);
8727 SDValue MFFS = DAG.getNode(PPCISD::MFFS, dl, {MVT::f64, MVT::Other}, Chain);
8728 Chain = MFFS.getValue(1);
8729
8730 SDValue CWD;
8731 if (isTypeLegal(MVT::i64)) {
8732 CWD = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
8733 DAG.getNode(ISD::BITCAST, dl, MVT::i64, MFFS));
8734 } else {
8735 // Save FP register to stack slot
8736 int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false);
8737 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
8738 Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo());
8739
8740 // Load FP Control Word from low 32 bits of stack slot.
8741 assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) &&(static_cast <bool> (hasBigEndianPartOrdering(MVT::i64,
MF.getDataLayout()) && "Stack slot adjustment is valid only on big endian subtargets!"
) ? void (0) : __assert_fail ("hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) && \"Stack slot adjustment is valid only on big endian subtargets!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8742, __extension__ __PRETTY_FUNCTION__))
8742 "Stack slot adjustment is valid only on big endian subtargets!")(static_cast <bool> (hasBigEndianPartOrdering(MVT::i64,
MF.getDataLayout()) && "Stack slot adjustment is valid only on big endian subtargets!"
) ? void (0) : __assert_fail ("hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) && \"Stack slot adjustment is valid only on big endian subtargets!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8742, __extension__ __PRETTY_FUNCTION__))
;
8743 SDValue Four = DAG.getConstant(4, dl, PtrVT);
8744 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four);
8745 CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo());
8746 Chain = CWD.getValue(1);
8747 }
8748
8749 // Transform as necessary
8750 SDValue CWD1 =
8751 DAG.getNode(ISD::AND, dl, MVT::i32,
8752 CWD, DAG.getConstant(3, dl, MVT::i32));
8753 SDValue CWD2 =
8754 DAG.getNode(ISD::SRL, dl, MVT::i32,
8755 DAG.getNode(ISD::AND, dl, MVT::i32,
8756 DAG.getNode(ISD::XOR, dl, MVT::i32,
8757 CWD, DAG.getConstant(3, dl, MVT::i32)),
8758 DAG.getConstant(3, dl, MVT::i32)),
8759 DAG.getConstant(1, dl, MVT::i32));
8760
8761 SDValue RetVal =
8762 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2);
8763
8764 RetVal =
8765 DAG.getNode((VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND),
8766 dl, VT, RetVal);
8767
8768 return DAG.getMergeValues({RetVal, Chain}, dl);
8769}
8770
8771SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
8772 EVT VT = Op.getValueType();
8773 unsigned BitWidth = VT.getSizeInBits();
8774 SDLoc dl(Op);
8775 assert(Op.getNumOperands() == 3 &&(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SHL!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SHL!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8777, __extension__ __PRETTY_FUNCTION__))
8776 VT == Op.getOperand(1).getValueType() &&(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SHL!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SHL!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8777, __extension__ __PRETTY_FUNCTION__))
8777 "Unexpected SHL!")(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SHL!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SHL!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8777, __extension__ __PRETTY_FUNCTION__))
;
8778
8779 // Expand into a bunch of logical ops. Note that these ops
8780 // depend on the PPC behavior for oversized shift amounts.
8781 SDValue Lo = Op.getOperand(0);
8782 SDValue Hi = Op.getOperand(1);
8783 SDValue Amt = Op.getOperand(2);
8784 EVT AmtVT = Amt.getValueType();
8785
8786 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
8787 DAG.getConstant(BitWidth, dl, AmtVT), Amt);
8788 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt);
8789 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1);
8790 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3);
8791 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
8792 DAG.getConstant(-BitWidth, dl, AmtVT));
8793 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5);
8794 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
8795 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt);
8796 SDValue OutOps[] = { OutLo, OutHi };
8797 return DAG.getMergeValues(OutOps, dl);
8798}
8799
8800SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
8801 EVT VT = Op.getValueType();
8802 SDLoc dl(Op);
8803 unsigned BitWidth = VT.getSizeInBits();
8804 assert(Op.getNumOperands() == 3 &&(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SRL!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SRL!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8806, __extension__ __PRETTY_FUNCTION__))
8805 VT == Op.getOperand(1).getValueType() &&(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SRL!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SRL!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8806, __extension__ __PRETTY_FUNCTION__))
8806 "Unexpected SRL!")(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SRL!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SRL!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8806, __extension__ __PRETTY_FUNCTION__))
;
8807
8808 // Expand into a bunch of logical ops. Note that these ops
8809 // depend on the PPC behavior for oversized shift amounts.
8810 SDValue Lo = Op.getOperand(0);
8811 SDValue Hi = Op.getOperand(1);
8812 SDValue Amt = Op.getOperand(2);
8813 EVT AmtVT = Amt.getValueType();
8814
8815 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
8816 DAG.getConstant(BitWidth, dl, AmtVT), Amt);
8817 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
8818 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
8819 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
8820 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
8821 DAG.getConstant(-BitWidth, dl, AmtVT));
8822 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5);
8823 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
8824 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt);
8825 SDValue OutOps[] = { OutLo, OutHi };
8826 return DAG.getMergeValues(OutOps, dl);
8827}
8828
8829SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
8830 SDLoc dl(Op);
8831 EVT VT = Op.getValueType();
8832 unsigned BitWidth = VT.getSizeInBits();
8833 assert(Op.getNumOperands() == 3 &&(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SRA!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SRA!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8835, __extension__ __PRETTY_FUNCTION__))
8834 VT == Op.getOperand(1).getValueType() &&(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SRA!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SRA!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8835, __extension__ __PRETTY_FUNCTION__))
8835 "Unexpected SRA!")(static_cast <bool> (Op.getNumOperands() == 3 &&
VT == Op.getOperand(1).getValueType() && "Unexpected SRA!"
) ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && \"Unexpected SRA!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 8835, __extension__ __PRETTY_FUNCTION__))
;
8836
8837 // Expand into a bunch of logical ops, followed by a select_cc.
8838 SDValue Lo = Op.getOperand(0);
8839 SDValue Hi = Op.getOperand(1);
8840 SDValue Amt = Op.getOperand(2);
8841 EVT AmtVT = Amt.getValueType();
8842
8843 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
8844 DAG.getConstant(BitWidth, dl, AmtVT), Amt);
8845 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
8846 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
8847 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
8848 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
8849 DAG.getConstant(-BitWidth, dl, AmtVT));
8850 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5);
8851 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt);
8852 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT),
8853 Tmp4, Tmp6, ISD::SETLE);
8854 SDValue OutOps[] = { OutLo, OutHi };
8855 return DAG.getMergeValues(OutOps, dl);
8856}
8857
8858SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op,
8859 SelectionDAG &DAG) const {
8860 SDLoc dl(Op);
8861 EVT VT = Op.getValueType();
8862 unsigned BitWidth = VT.getSizeInBits();
8863
8864 bool IsFSHL = Op.getOpcode() == ISD::FSHL;
8865 SDValue X = Op.getOperand(0);
8866 SDValue Y = Op.getOperand(1);
8867 SDValue Z = Op.getOperand(2);
8868 EVT AmtVT = Z.getValueType();
8869
8870 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
8871 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
8872 // This is simpler than TargetLowering::expandFunnelShift because we can rely
8873 // on PowerPC shift by BW being well defined.
8874 Z = DAG.getNode(ISD::AND, dl, AmtVT, Z,
8875 DAG.getConstant(BitWidth - 1, dl, AmtVT));
8876 SDValue SubZ =
8877 DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Z);
8878 X = DAG.getNode(PPCISD::SHL, dl, VT, X, IsFSHL ? Z : SubZ);
8879 Y = DAG.getNode(PPCISD::SRL, dl, VT, Y, IsFSHL ? SubZ : Z);
8880 return DAG.getNode(ISD::OR, dl, VT, X, Y);
8881}
8882
8883//===----------------------------------------------------------------------===//
8884// Vector related lowering.
8885//
8886
8887/// getCanonicalConstSplat - Build a canonical splat immediate of Val with an
8888/// element size of SplatSize. Cast the result to VT.
8889static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT,
8890 SelectionDAG &DAG, const SDLoc &dl) {
8891 static const MVT VTys[] = { // canonical VT to use for each size.
8892 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
8893 };
8894
8895 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
8896
8897 // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize.
8898 if (Val == ((1LLU << (SplatSize * 8)) - 1)) {
8899 SplatSize = 1;
8900 Val = 0xFF;
8901 }
8902
8903 EVT CanonicalVT = VTys[SplatSize-1];
8904
8905 // Build a canonical splat for this value.
8906 return DAG.getBitcast(ReqVT, DAG.getConstant(Val, dl, CanonicalVT));
8907}
8908
8909/// BuildIntrinsicOp - Return a unary operator intrinsic node with the
8910/// specified intrinsic ID.
8911static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
8912 const SDLoc &dl, EVT DestVT = MVT::Other) {
8913 if (DestVT == MVT::Other) DestVT = Op.getValueType();
8914 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
8915 DAG.getConstant(IID, dl, MVT::i32), Op);
8916}
8917
8918/// BuildIntrinsicOp - Return a binary operator intrinsic node with the
8919/// specified intrinsic ID.
8920static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
8921 SelectionDAG &DAG, const SDLoc &dl,
8922 EVT DestVT = MVT::Other) {
8923 if (DestVT == MVT::Other) DestVT = LHS.getValueType();
8924 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
8925 DAG.getConstant(IID, dl, MVT::i32), LHS, RHS);
8926}
8927
8928/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
8929/// specified intrinsic ID.
8930static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
8931 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl,
8932 EVT DestVT = MVT::Other) {
8933 if (DestVT == MVT::Other) DestVT = Op0.getValueType();
8934 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
8935 DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2);
8936}
8937
8938/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified
8939/// amount. The result has the specified value type.
8940static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
8941 SelectionDAG &DAG, const SDLoc &dl) {
8942 // Force LHS/RHS to be the right type.
8943 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS);
8944 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS);
8945
8946 int Ops[16];
8947 for (unsigned i = 0; i != 16; ++i)
8948 Ops[i] = i + Amt;
8949 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops);
8950 return DAG.getNode(ISD::BITCAST, dl, VT, T);
8951}
8952
8953/// Do we have an efficient pattern in a .td file for this node?
8954///
8955/// \param V - pointer to the BuildVectorSDNode being matched
8956/// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves?
8957///
8958/// There are some patterns where it is beneficial to keep a BUILD_VECTOR
8959/// node as a BUILD_VECTOR node rather than expanding it. The patterns where
8960/// the opposite is true (expansion is beneficial) are:
8961/// - The node builds a vector out of integers that are not 32 or 64-bits
8962/// - The node builds a vector out of constants
8963/// - The node is a "load-and-splat"
8964/// In all other cases, we will choose to keep the BUILD_VECTOR.
8965static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V,
8966 bool HasDirectMove,
8967 bool HasP8Vector) {
8968 EVT VecVT = V->getValueType(0);
8969 bool RightType = VecVT == MVT::v2f64 ||
8970 (HasP8Vector && VecVT == MVT::v4f32) ||
8971 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32));
8972 if (!RightType)
8973 return false;
8974
8975 bool IsSplat = true;
8976 bool IsLoad = false;
8977 SDValue Op0 = V->getOperand(0);
8978
8979 // This function is called in a block that confirms the node is not a constant
8980 // splat. So a constant BUILD_VECTOR here means the vector is built out of
8981 // different constants.
8982 if (V->isConstant())
8983 return false;
8984 for (int i = 0, e = V->getNumOperands(); i < e; ++i) {
8985 if (V->getOperand(i).isUndef())
8986 return false;
8987 // We want to expand nodes that represent load-and-splat even if the
8988 // loaded value is a floating point truncation or conversion to int.
8989 if (V->getOperand(i).getOpcode() == ISD::LOAD ||
8990 (V->getOperand(i).getOpcode() == ISD::FP_ROUND &&
8991 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) ||
8992 (V->getOperand(i).getOpcode() == ISD::FP_TO_SINT &&
8993 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) ||
8994 (V->getOperand(i).getOpcode() == ISD::FP_TO_UINT &&
8995 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD))
8996 IsLoad = true;
8997 // If the operands are different or the input is not a load and has more
8998 // uses than just this BV node, then it isn't a splat.
8999 if (V->getOperand(i) != Op0 ||
9000 (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode())))
9001 IsSplat = false;
9002 }
9003 return !(IsSplat && IsLoad);
9004}
9005
9006// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128.
9007SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
9008
9009 SDLoc dl(Op);
9010 SDValue Op0 = Op->getOperand(0);
9011
9012 if ((Op.getValueType() != MVT::f128) ||
9013 (Op0.getOpcode() != ISD::BUILD_PAIR) ||
9014 (Op0.getOperand(0).getValueType() != MVT::i64) ||
9015 (Op0.getOperand(1).getValueType() != MVT::i64))
9016 return SDValue();
9017
9018 return DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128, Op0.getOperand(0),
9019 Op0.getOperand(1));
9020}
9021
9022static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) {
9023 const SDValue *InputLoad = &Op;
9024 if (InputLoad->getOpcode() == ISD::BITCAST)
9025 InputLoad = &InputLoad->getOperand(0);
9026 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR ||
9027 InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) {
9028 IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED;
9029 InputLoad = &InputLoad->getOperand(0);
9030 }
9031 if (InputLoad->getOpcode() != ISD::LOAD)
9032 return nullptr;
9033 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
9034 return ISD::isNormalLoad(LD) ? InputLoad : nullptr;
9035}
9036
9037// Convert the argument APFloat to a single precision APFloat if there is no
9038// loss in information during the conversion to single precision APFloat and the
9039// resulting number is not a denormal number. Return true if successful.
9040bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) {
9041 APFloat APFloatToConvert = ArgAPFloat;
9042 bool LosesInfo = true;
9043 APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
9044 &LosesInfo);
9045 bool Success = (!LosesInfo && !APFloatToConvert.isDenormal());
9046 if (Success)
9047 ArgAPFloat = APFloatToConvert;
9048 return Success;
9049}
9050
9051// Bitcast the argument APInt to a double and convert it to a single precision
9052// APFloat, bitcast the APFloat to an APInt and assign it to the original
9053// argument if there is no loss in information during the conversion from
9054// double to single precision APFloat and the resulting number is not a denormal
9055// number. Return true if successful.
9056bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
9057 double DpValue = ArgAPInt.bitsToDouble();
9058 APFloat APFloatDp(DpValue);
9059 bool Success = convertToNonDenormSingle(APFloatDp);
9060 if (Success)
9061 ArgAPInt = APFloatDp.bitcastToAPInt();
9062 return Success;
9063}
9064
9065// Nondestructive check for convertTonNonDenormSingle.
9066bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) {
9067 // Only convert if it loses info, since XXSPLTIDP should
9068 // handle the other case.
9069 APFloat APFloatToConvert = ArgAPFloat;
9070 bool LosesInfo = true;
9071 APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
9072 &LosesInfo);
9073
9074 return (!LosesInfo && !APFloatToConvert.isDenormal());
9075}
9076
9077// If this is a case we can't handle, return null and let the default
9078// expansion code take care of it. If we CAN select this case, and if it
9079// selects to a single instruction, return Op. Otherwise, if we can codegen
9080// this case more efficiently than a constant pool load, lower it to the
9081// sequence of ops that should be used.
9082SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
9083 SelectionDAG &DAG) const {
9084 SDLoc dl(Op);
9085 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
9086 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR")(static_cast <bool> (BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"
) ? void (0) : __assert_fail ("BVN && \"Expected a BuildVectorSDNode in LowerBUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9086, __extension__ __PRETTY_FUNCTION__))
;
9087
9088 // Check if this is a splat of a constant value.
9089 APInt APSplatBits, APSplatUndef;
9090 unsigned SplatBitSize;
9091 bool HasAnyUndefs;
9092 bool BVNIsConstantSplat =
9093 BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
9094 HasAnyUndefs, 0, !Subtarget.isLittleEndian());
9095
9096 // If it is a splat of a double, check if we can shrink it to a 32 bit
9097 // non-denormal float which when converted back to double gives us the same
9098 // double. This is to exploit the XXSPLTIDP instruction.
9099 // If we lose precision, we use XXSPLTI32DX.
9100 if (BVNIsConstantSplat && (SplatBitSize == 64) &&
9101 Subtarget.hasPrefixInstrs()) {
9102 // Check the type first to short-circuit so we don't modify APSplatBits if
9103 // this block isn't executed.
9104 if ((Op->getValueType(0) == MVT::v2f64) &&
9105 convertToNonDenormSingle(APSplatBits)) {
9106 SDValue SplatNode = DAG.getNode(
9107 PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64,
9108 DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32));
9109 return DAG.getBitcast(Op.getValueType(), SplatNode);
9110 } else {
9111 // We may lose precision, so we have to use XXSPLTI32DX.
9112
9113 uint32_t Hi =
9114 (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32);
9115 uint32_t Lo =
9116 (uint32_t)(APSplatBits.getZExtValue() & 0xFFFFFFFF);
9117 SDValue SplatNode = DAG.getUNDEF(MVT::v2i64);
9118
9119 if (!Hi || !Lo)
9120 // If either load is 0, then we should generate XXLXOR to set to 0.
9121 SplatNode = DAG.getTargetConstant(0, dl, MVT::v2i64);
9122
9123 if (Hi)
9124 SplatNode = DAG.getNode(
9125 PPCISD::XXSPLTI32DX, dl, MVT::v2i64, SplatNode,
9126 DAG.getTargetConstant(0, dl, MVT::i32),
9127 DAG.getTargetConstant(Hi, dl, MVT::i32));
9128
9129 if (Lo)
9130 SplatNode =
9131 DAG.getNode(PPCISD::XXSPLTI32DX, dl, MVT::v2i64, SplatNode,
9132 DAG.getTargetConstant(1, dl, MVT::i32),
9133 DAG.getTargetConstant(Lo, dl, MVT::i32));
9134
9135 return DAG.getBitcast(Op.getValueType(), SplatNode);
9136 }
9137 }
9138
9139 if (!BVNIsConstantSplat || SplatBitSize > 32) {
9140
9141 bool IsPermutedLoad = false;
9142 const SDValue *InputLoad =
9143 getNormalLoadInput(Op.getOperand(0), IsPermutedLoad);
9144 // Handle load-and-splat patterns as we have instructions that will do this
9145 // in one go.
9146 if (InputLoad && DAG.isSplatValue(Op, true)) {
9147 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
9148
9149 // We have handling for 4 and 8 byte elements.
9150 unsigned ElementSize = LD->getMemoryVT().getScalarSizeInBits();
9151
9152 // Checking for a single use of this load, we have to check for vector
9153 // width (128 bits) / ElementSize uses (since each operand of the
9154 // BUILD_VECTOR is a separate use of the value.
9155 unsigned NumUsesOfInputLD = 128 / ElementSize;
9156 for (SDValue BVInOp : Op->ops())
9157 if (BVInOp.isUndef())
9158 NumUsesOfInputLD--;
9159 assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?")(static_cast <bool> (NumUsesOfInputLD > 0 &&
"No uses of input LD of a build_vector?") ? void (0) : __assert_fail
("NumUsesOfInputLD > 0 && \"No uses of input LD of a build_vector?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9159, __extension__ __PRETTY_FUNCTION__))
;
9160 if (InputLoad->getNode()->hasNUsesOfValue(NumUsesOfInputLD, 0) &&
9161 ((Subtarget.hasVSX() && ElementSize == 64) ||
9162 (Subtarget.hasP9Vector() && ElementSize == 32))) {
9163 SDValue Ops[] = {
9164 LD->getChain(), // Chain
9165 LD->getBasePtr(), // Ptr
9166 DAG.getValueType(Op.getValueType()) // VT
9167 };
9168 SDValue LdSplt = DAG.getMemIntrinsicNode(
9169 PPCISD::LD_SPLAT, dl, DAG.getVTList(Op.getValueType(), MVT::Other),
9170 Ops, LD->getMemoryVT(), LD->getMemOperand());
9171 // Replace all uses of the output chain of the original load with the
9172 // output chain of the new load.
9173 DAG.ReplaceAllUsesOfValueWith(InputLoad->getValue(1),
9174 LdSplt.getValue(1));
9175 return LdSplt;
9176 }
9177 }
9178
9179 // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to
9180 // 32-bits can be lowered to VSX instructions under certain conditions.
9181 // Without VSX, there is no pattern more efficient than expanding the node.
9182 if (Subtarget.hasVSX() && Subtarget.isPPC64() &&
9183 haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(),
9184 Subtarget.hasP8Vector()))
9185 return Op;
9186 return SDValue();
9187 }
9188
9189 uint64_t SplatBits = APSplatBits.getZExtValue();
9190 uint64_t SplatUndef = APSplatUndef.getZExtValue();
9191 unsigned SplatSize = SplatBitSize / 8;
9192
9193 // First, handle single instruction cases.
9194
9195 // All zeros?
9196 if (SplatBits == 0) {
9197 // Canonicalize all zero vectors to be v4i32.
9198 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
9199 SDValue Z = DAG.getConstant(0, dl, MVT::v4i32);
9200 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z);
9201 }
9202 return Op;
9203 }
9204
9205 // We have XXSPLTIW for constant splats four bytes wide.
9206 // Given vector length is a multiple of 4, 2-byte splats can be replaced
9207 // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to
9208 // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be
9209 // turned into a 4-byte splat of 0xABABABAB.
9210 if (Subtarget.hasPrefixInstrs() && SplatSize == 2)
9211 return getCanonicalConstSplat(SplatBits | (SplatBits << 16), SplatSize * 2,
9212 Op.getValueType(), DAG, dl);
9213
9214 if (Subtarget.hasPrefixInstrs() && SplatSize == 4)
9215 return getCanonicalConstSplat(SplatBits, SplatSize, Op.getValueType(), DAG,
9216 dl);
9217
9218 // We have XXSPLTIB for constant splats one byte wide.
9219 if (Subtarget.hasP9Vector() && SplatSize == 1)
9220 return getCanonicalConstSplat(SplatBits, SplatSize, Op.getValueType(), DAG,
9221 dl);
9222
9223 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
9224 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >>
9225 (32-SplatBitSize));
9226 if (SextVal >= -16 && SextVal <= 15)
9227 return getCanonicalConstSplat(SextVal, SplatSize, Op.getValueType(), DAG,
9228 dl);
9229
9230 // Two instruction sequences.
9231
9232 // If this value is in the range [-32,30] and is even, use:
9233 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
9234 // If this value is in the range [17,31] and is odd, use:
9235 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16)
9236 // If this value is in the range [-31,-17] and is odd, use:
9237 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16)
9238 // Note the last two are three-instruction sequences.
9239 if (SextVal >= -32 && SextVal <= 31) {
9240 // To avoid having these optimizations undone by constant folding,
9241 // we convert to a pseudo that will be expanded later into one of
9242 // the above forms.
9243 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32);
9244 EVT VT = (SplatSize == 1 ? MVT::v16i8 :
9245 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
9246 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32);
9247 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize);
9248 if (VT == Op.getValueType())
9249 return RetVal;
9250 else
9251 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal);
9252 }
9253
9254 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is
9255 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important
9256 // for fneg/fabs.
9257 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
9258 // Make -1 and vspltisw -1:
9259 SDValue OnesV = getCanonicalConstSplat(-1, 4, MVT::v4i32, DAG, dl);
9260
9261 // Make the VSLW intrinsic, computing 0x8000_0000.
9262 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV,
9263 OnesV, DAG, dl);
9264
9265 // xor by OnesV to invert it.
9266 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV);
9267 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
9268 }
9269
9270 // Check to see if this is a wide variety of vsplti*, binop self cases.
9271 static const signed char SplatCsts[] = {
9272 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7,
9273 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16
9274 };
9275
9276 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) {
9277 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for
9278 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1'
9279 int i = SplatCsts[idx];
9280
9281 // Figure out what shift amount will be used by altivec if shifted by i in
9282 // this splat size.
9283 unsigned TypeShiftAmt = i & (SplatBitSize-1);
9284
9285 // vsplti + shl self.
9286 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
9287 SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
9288 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9289 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
9290 Intrinsic::ppc_altivec_vslw
9291 };
9292 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
9293 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
9294 }
9295
9296 // vsplti + srl self.
9297 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
9298 SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
9299 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9300 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
9301 Intrinsic::ppc_altivec_vsrw
9302 };
9303 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
9304 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
9305 }
9306
9307 // vsplti + rol self.
9308 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
9309 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
9310 SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl);
9311 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9312 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
9313 Intrinsic::ppc_altivec_vrlw
9314 };
9315 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
9316 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
9317 }
9318
9319 // t = vsplti c, result = vsldoi t, t, 1
9320 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
9321 SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl);
9322 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
9323 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
9324 }
9325 // t = vsplti c, result = vsldoi t, t, 2
9326 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
9327 SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl);
9328 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
9329 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
9330 }
9331 // t = vsplti c, result = vsldoi t, t, 3
9332 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
9333 SDValue T = getCanonicalConstSplat(i, SplatSize, MVT::v16i8, DAG, dl);
9334 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
9335 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl);
9336 }
9337 }
9338
9339 return SDValue();
9340}
9341
9342/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
9343/// the specified operations to build the shuffle.
9344static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
9345 SDValue RHS, SelectionDAG &DAG,
9346 const SDLoc &dl) {
9347 unsigned OpNum = (PFEntry >> 26) & 0x0F;
9348 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
9349 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
9350
9351 enum {
9352 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
9353 OP_VMRGHW,
9354 OP_VMRGLW,
9355 OP_VSPLTISW0,
9356 OP_VSPLTISW1,
9357 OP_VSPLTISW2,
9358 OP_VSPLTISW3,
9359 OP_VSLDOI4,
9360 OP_VSLDOI8,
9361 OP_VSLDOI12
9362 };
9363
9364 if (OpNum == OP_COPY) {
9365 if (LHSID == (1*9+2)*9+3) return LHS;
9366 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!")(static_cast <bool> (LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!") ? void (0) : __assert_fail ("LHSID == ((4*9+5)*9+6)*9+7 && \"Illegal OP_COPY!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9366, __extension__ __PRETTY_FUNCTION__))
;
9367 return RHS;
9368 }
9369
9370 SDValue OpLHS, OpRHS;
9371 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
9372 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
9373
9374 int ShufIdxs[16];
9375 switch (OpNum) {
9376 default: llvm_unreachable("Unknown i32 permute!")::llvm::llvm_unreachable_internal("Unknown i32 permute!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9376)
;
9377 case OP_VMRGHW:
9378 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3;
9379 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19;
9380 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7;
9381 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23;
9382 break;
9383 case OP_VMRGLW:
9384 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11;
9385 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27;
9386 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15;
9387 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31;
9388 break;
9389 case OP_VSPLTISW0:
9390 for (unsigned i = 0; i != 16; ++i)
9391 ShufIdxs[i] = (i&3)+0;
9392 break;
9393 case OP_VSPLTISW1:
9394 for (unsigned i = 0; i != 16; ++i)
9395 ShufIdxs[i] = (i&3)+4;
9396 break;
9397 case OP_VSPLTISW2:
9398 for (unsigned i = 0; i != 16; ++i)
9399 ShufIdxs[i] = (i&3)+8;
9400 break;
9401 case OP_VSPLTISW3:
9402 for (unsigned i = 0; i != 16; ++i)
9403 ShufIdxs[i] = (i&3)+12;
9404 break;
9405 case OP_VSLDOI4:
9406 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl);
9407 case OP_VSLDOI8:
9408 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl);
9409 case OP_VSLDOI12:
9410 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl);
9411 }
9412 EVT VT = OpLHS.getValueType();
9413 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS);
9414 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS);
9415 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs);
9416 return DAG.getNode(ISD::BITCAST, dl, VT, T);
9417}
9418
9419/// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled
9420/// by the VINSERTB instruction introduced in ISA 3.0, else just return default
9421/// SDValue.
9422SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N,
9423 SelectionDAG &DAG) const {
9424 const unsigned BytesInVector = 16;
9425 bool IsLE = Subtarget.isLittleEndian();
9426 SDLoc dl(N);
9427 SDValue V1 = N->getOperand(0);
9428 SDValue V2 = N->getOperand(1);
9429 unsigned ShiftElts = 0, InsertAtByte = 0;
9430 bool Swap = false;
9431
9432 // Shifts required to get the byte we want at element 7.
9433 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1,
9434 0, 15, 14, 13, 12, 11, 10, 9};
9435 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0,
9436 1, 2, 3, 4, 5, 6, 7, 8};
9437
9438 ArrayRef<int> Mask = N->getMask();
9439 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
9440
9441 // For each mask element, find out if we're just inserting something
9442 // from V2 into V1 or vice versa.
9443 // Possible permutations inserting an element from V2 into V1:
9444 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
9445 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
9446 // ...
9447 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X
9448 // Inserting from V1 into V2 will be similar, except mask range will be
9449 // [16,31].
9450
9451 bool FoundCandidate = false;
9452 // If both vector operands for the shuffle are the same vector, the mask
9453 // will contain only elements from the first one and the second one will be
9454 // undef.
9455 unsigned VINSERTBSrcElem = IsLE ? 8 : 7;
9456 // Go through the mask of half-words to find an element that's being moved
9457 // from one vector to the other.
9458 for (unsigned i = 0; i < BytesInVector; ++i) {
9459 unsigned CurrentElement = Mask[i];
9460 // If 2nd operand is undefined, we should only look for element 7 in the
9461 // Mask.
9462 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem)
9463 continue;
9464
9465 bool OtherElementsInOrder = true;
9466 // Examine the other elements in the Mask to see if they're in original
9467 // order.
9468 for (unsigned j = 0; j < BytesInVector; ++j) {
9469 if (j == i)
9470 continue;
9471 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be
9472 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined,
9473 // in which we always assume we're always picking from the 1st operand.
9474 int MaskOffset =
9475 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0;
9476 if (Mask[j] != OriginalOrder[j] + MaskOffset) {
9477 OtherElementsInOrder = false;
9478 break;
9479 }
9480 }
9481 // If other elements are in original order, we record the number of shifts
9482 // we need to get the element we want into element 7. Also record which byte
9483 // in the vector we should insert into.
9484 if (OtherElementsInOrder) {
9485 // If 2nd operand is undefined, we assume no shifts and no swapping.
9486 if (V2.isUndef()) {
9487 ShiftElts = 0;
9488 Swap = false;
9489 } else {
9490 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4.
9491 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF]
9492 : BigEndianShifts[CurrentElement & 0xF];
9493 Swap = CurrentElement < BytesInVector;
9494 }
9495 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i;
9496 FoundCandidate = true;
9497 break;
9498 }
9499 }
9500
9501 if (!FoundCandidate)
9502 return SDValue();
9503
9504 // Candidate found, construct the proper SDAG sequence with VINSERTB,
9505 // optionally with VECSHL if shift is required.
9506 if (Swap)
9507 std::swap(V1, V2);
9508 if (V2.isUndef())
9509 V2 = V1;
9510 if (ShiftElts) {
9511 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2,
9512 DAG.getConstant(ShiftElts, dl, MVT::i32));
9513 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, Shl,
9514 DAG.getConstant(InsertAtByte, dl, MVT::i32));
9515 }
9516 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, V2,
9517 DAG.getConstant(InsertAtByte, dl, MVT::i32));
9518}
9519
9520/// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled
9521/// by the VINSERTH instruction introduced in ISA 3.0, else just return default
9522/// SDValue.
9523SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
9524 SelectionDAG &DAG) const {
9525 const unsigned NumHalfWords = 8;
9526 const unsigned BytesInVector = NumHalfWords * 2;
9527 // Check that the shuffle is on half-words.
9528 if (!isNByteElemShuffleMask(N, 2, 1))
9529 return SDValue();
9530
9531 bool IsLE = Subtarget.isLittleEndian();
9532 SDLoc dl(N);
9533 SDValue V1 = N->getOperand(0);
9534 SDValue V2 = N->getOperand(1);
9535 unsigned ShiftElts = 0, InsertAtByte = 0;
9536 bool Swap = false;
9537
9538 // Shifts required to get the half-word we want at element 3.
9539 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5};
9540 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4};
9541
9542 uint32_t Mask = 0;
9543 uint32_t OriginalOrderLow = 0x1234567;
9544 uint32_t OriginalOrderHigh = 0x89ABCDEF;
9545 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a
9546 // 32-bit space, only need 4-bit nibbles per element.
9547 for (unsigned i = 0; i < NumHalfWords; ++i) {
9548 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
9549 Mask |= ((uint32_t)(N->getMaskElt(i * 2) / 2) << MaskShift);
9550 }
9551
9552 // For each mask element, find out if we're just inserting something
9553 // from V2 into V1 or vice versa. Possible permutations inserting an element
9554 // from V2 into V1:
9555 // X, 1, 2, 3, 4, 5, 6, 7
9556 // 0, X, 2, 3, 4, 5, 6, 7
9557 // 0, 1, X, 3, 4, 5, 6, 7
9558 // 0, 1, 2, X, 4, 5, 6, 7
9559 // 0, 1, 2, 3, X, 5, 6, 7
9560 // 0, 1, 2, 3, 4, X, 6, 7
9561 // 0, 1, 2, 3, 4, 5, X, 7
9562 // 0, 1, 2, 3, 4, 5, 6, X
9563 // Inserting from V1 into V2 will be similar, except mask range will be [8,15].
9564
9565 bool FoundCandidate = false;
9566 // Go through the mask of half-words to find an element that's being moved
9567 // from one vector to the other.
9568 for (unsigned i = 0; i < NumHalfWords; ++i) {
9569 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
9570 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF;
9571 uint32_t MaskOtherElts = ~(0xF << MaskShift);
9572 uint32_t TargetOrder = 0x0;
9573
9574 // If both vector operands for the shuffle are the same vector, the mask
9575 // will contain only elements from the first one and the second one will be
9576 // undef.
9577 if (V2.isUndef()) {
9578 ShiftElts = 0;
9579 unsigned VINSERTHSrcElem = IsLE ? 4 : 3;
9580 TargetOrder = OriginalOrderLow;
9581 Swap = false;
9582 // Skip if not the correct element or mask of other elements don't equal
9583 // to our expected order.
9584 if (MaskOneElt == VINSERTHSrcElem &&
9585 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
9586 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
9587 FoundCandidate = true;
9588 break;
9589 }
9590 } else { // If both operands are defined.
9591 // Target order is [8,15] if the current mask is between [0,7].
9592 TargetOrder =
9593 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow;
9594 // Skip if mask of other elements don't equal our expected order.
9595 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
9596 // We only need the last 3 bits for the number of shifts.
9597 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7]
9598 : BigEndianShifts[MaskOneElt & 0x7];
9599 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
9600 Swap = MaskOneElt < NumHalfWords;
9601 FoundCandidate = true;
9602 break;
9603 }
9604 }
9605 }
9606
9607 if (!FoundCandidate)
9608 return SDValue();
9609
9610 // Candidate found, construct the proper SDAG sequence with VINSERTH,
9611 // optionally with VECSHL if shift is required.
9612 if (Swap)
9613 std::swap(V1, V2);
9614 if (V2.isUndef())
9615 V2 = V1;
9616 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
9617 if (ShiftElts) {
9618 // Double ShiftElts because we're left shifting on v16i8 type.
9619 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2,
9620 DAG.getConstant(2 * ShiftElts, dl, MVT::i32));
9621 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl);
9622 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2,
9623 DAG.getConstant(InsertAtByte, dl, MVT::i32));
9624 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
9625 }
9626 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2);
9627 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2,
9628 DAG.getConstant(InsertAtByte, dl, MVT::i32));
9629 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
9630}
9631
9632/// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
9633/// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise
9634/// return the default SDValue.
9635SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN,
9636 SelectionDAG &DAG) const {
9637 // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles
9638 // to v16i8. Peek through the bitcasts to get the actual operands.
9639 SDValue LHS = peekThroughBitcasts(SVN->getOperand(0));
9640 SDValue RHS = peekThroughBitcasts(SVN->getOperand(1));
9641
9642 auto ShuffleMask = SVN->getMask();
9643 SDValue VecShuffle(SVN, 0);
9644 SDLoc DL(SVN);
9645
9646 // Check that we have a four byte shuffle.
9647 if (!isNByteElemShuffleMask(SVN, 4, 1))
9648 return SDValue();
9649
9650 // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx.
9651 if (RHS->getOpcode() != ISD::BUILD_VECTOR) {
9652 std::swap(LHS, RHS);
9653 VecShuffle = DAG.getCommutedVectorShuffle(*SVN);
9654 ShuffleMask = cast<ShuffleVectorSDNode>(VecShuffle)->getMask();
9655 }
9656
9657 // Ensure that the RHS is a vector of constants.
9658 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode());
9659 if (!BVN)
9660 return SDValue();
9661
9662 // Check if RHS is a splat of 4-bytes (or smaller).
9663 APInt APSplatValue, APSplatUndef;
9664 unsigned SplatBitSize;
9665 bool HasAnyUndefs;
9666 if (!BVN->isConstantSplat(APSplatValue, APSplatUndef, SplatBitSize,
9667 HasAnyUndefs, 0, !Subtarget.isLittleEndian()) ||
9668 SplatBitSize > 32)
9669 return SDValue();
9670
9671 // Check that the shuffle mask matches the semantics of XXSPLTI32DX.
9672 // The instruction splats a constant C into two words of the source vector
9673 // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }.
9674 // Thus we check that the shuffle mask is the equivalent of
9675 // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively.
9676 // Note: the check above of isNByteElemShuffleMask() ensures that the bytes
9677 // within each word are consecutive, so we only need to check the first byte.
9678 SDValue Index;
9679 bool IsLE = Subtarget.isLittleEndian();
9680 if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) &&
9681 (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 &&
9682 ShuffleMask[4] > 15 && ShuffleMask[12] > 15))
9683 Index = DAG.getTargetConstant(IsLE ? 0 : 1, DL, MVT::i32);
9684 else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) &&
9685 (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 &&
9686 ShuffleMask[0] > 15 && ShuffleMask[8] > 15))
9687 Index = DAG.getTargetConstant(IsLE ? 1 : 0, DL, MVT::i32);
9688 else
9689 return SDValue();
9690
9691 // If the splat is narrower than 32-bits, we need to get the 32-bit value
9692 // for XXSPLTI32DX.
9693 unsigned SplatVal = APSplatValue.getZExtValue();
9694 for (; SplatBitSize < 32; SplatBitSize <<= 1)
9695 SplatVal |= (SplatVal << SplatBitSize);
9696
9697 SDValue SplatNode = DAG.getNode(
9698 PPCISD::XXSPLTI32DX, DL, MVT::v2i64, DAG.getBitcast(MVT::v2i64, LHS),
9699 Index, DAG.getTargetConstant(SplatVal, DL, MVT::i32));
9700 return DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, SplatNode);
9701}
9702
9703/// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
9704/// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
9705/// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
9706/// i.e (or (shl x, C1), (srl x, 128-C1)).
9707SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
9708 assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL")(static_cast <bool> (Op.getOpcode() == ISD::ROTL &&
"Should only be called for ISD::ROTL") ? void (0) : __assert_fail
("Op.getOpcode() == ISD::ROTL && \"Should only be called for ISD::ROTL\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9708, __extension__ __PRETTY_FUNCTION__))
;
9709 assert(Op.getValueType() == MVT::v1i128 &&(static_cast <bool> (Op.getValueType() == MVT::v1i128 &&
"Only set v1i128 as custom, other type shouldn't reach here!"
) ? void (0) : __assert_fail ("Op.getValueType() == MVT::v1i128 && \"Only set v1i128 as custom, other type shouldn't reach here!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9710, __extension__ __PRETTY_FUNCTION__))
9710 "Only set v1i128 as custom, other type shouldn't reach here!")(static_cast <bool> (Op.getValueType() == MVT::v1i128 &&
"Only set v1i128 as custom, other type shouldn't reach here!"
) ? void (0) : __assert_fail ("Op.getValueType() == MVT::v1i128 && \"Only set v1i128 as custom, other type shouldn't reach here!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9710, __extension__ __PRETTY_FUNCTION__))
;
9711 SDLoc dl(Op);
9712 SDValue N0 = peekThroughBitcasts(Op.getOperand(0));
9713 SDValue N1 = peekThroughBitcasts(Op.getOperand(1));
9714 unsigned SHLAmt = N1.getConstantOperandVal(0);
9715 if (SHLAmt % 8 == 0) {
9716 SmallVector<int, 16> Mask(16, 0);
9717 std::iota(Mask.begin(), Mask.end(), 0);
9718 std::rotate(Mask.begin(), Mask.begin() + SHLAmt / 8, Mask.end());
9719 if (SDValue Shuffle =
9720 DAG.getVectorShuffle(MVT::v16i8, dl, DAG.getBitcast(MVT::v16i8, N0),
9721 DAG.getUNDEF(MVT::v16i8), Mask))
9722 return DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, Shuffle);
9723 }
9724 SDValue ArgVal = DAG.getBitcast(MVT::i128, N0);
9725 SDValue SHLOp = DAG.getNode(ISD::SHL, dl, MVT::i128, ArgVal,
9726 DAG.getConstant(SHLAmt, dl, MVT::i32));
9727 SDValue SRLOp = DAG.getNode(ISD::SRL, dl, MVT::i128, ArgVal,
9728 DAG.getConstant(128 - SHLAmt, dl, MVT::i32));
9729 SDValue OROp = DAG.getNode(ISD::OR, dl, MVT::i128, SHLOp, SRLOp);
9730 return DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, OROp);
9731}
9732
9733/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
9734/// is a shuffle we can handle in a single instruction, return it. Otherwise,
9735/// return the code it can be lowered into. Worst case, it can always be
9736/// lowered into a vperm.
9737SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
9738 SelectionDAG &DAG) const {
9739 SDLoc dl(Op);
9740 SDValue V1 = Op.getOperand(0);
9741 SDValue V2 = Op.getOperand(1);
9742 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
9743
9744 // Any nodes that were combined in the target-independent combiner prior
9745 // to vector legalization will not be sent to the target combine. Try to
9746 // combine it here.
9747 if (SDValue NewShuffle = combineVectorShuffle(SVOp, DAG)) {
9748 if (!isa<ShuffleVectorSDNode>(NewShuffle))
9749 return NewShuffle;
9750 Op = NewShuffle;
9751 SVOp = cast<ShuffleVectorSDNode>(Op);
9752 V1 = Op.getOperand(0);
9753 V2 = Op.getOperand(1);
9754 }
9755 EVT VT = Op.getValueType();
9756 bool isLittleEndian = Subtarget.isLittleEndian();
9757
9758 unsigned ShiftElts, InsertAtByte;
9759 bool Swap = false;
9760
9761 // If this is a load-and-splat, we can do that with a single instruction
9762 // in some cases. However if the load has multiple uses, we don't want to
9763 // combine it because that will just produce multiple loads.
9764 bool IsPermutedLoad = false;
9765 const SDValue *InputLoad = getNormalLoadInput(V1, IsPermutedLoad);
9766 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
9767 (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) &&
9768 InputLoad->hasOneUse()) {
9769 bool IsFourByte = PPC::isSplatShuffleMask(SVOp, 4);
9770 int SplatIdx =
9771 PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG);
9772
9773 // The splat index for permuted loads will be in the left half of the vector
9774 // which is strictly wider than the loaded value by 8 bytes. So we need to
9775 // adjust the splat index to point to the correct address in memory.
9776 if (IsPermutedLoad) {
9777 assert((isLittleEndian || IsFourByte) &&(static_cast <bool> ((isLittleEndian || IsFourByte) &&
"Unexpected size for permuted load on big endian target") ? void
(0) : __assert_fail ("(isLittleEndian || IsFourByte) && \"Unexpected size for permuted load on big endian target\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9778, __extension__ __PRETTY_FUNCTION__))
9778 "Unexpected size for permuted load on big endian target")(static_cast <bool> ((isLittleEndian || IsFourByte) &&
"Unexpected size for permuted load on big endian target") ? void
(0) : __assert_fail ("(isLittleEndian || IsFourByte) && \"Unexpected size for permuted load on big endian target\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9778, __extension__ __PRETTY_FUNCTION__))
;
9779 SplatIdx += IsFourByte ? 2 : 1;
9780 assert((SplatIdx < (IsFourByte ? 4 : 2)) &&(static_cast <bool> ((SplatIdx < (IsFourByte ? 4 : 2
)) && "Splat of a value outside of the loaded memory"
) ? void (0) : __assert_fail ("(SplatIdx < (IsFourByte ? 4 : 2)) && \"Splat of a value outside of the loaded memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9781, __extension__ __PRETTY_FUNCTION__))
9781 "Splat of a value outside of the loaded memory")(static_cast <bool> ((SplatIdx < (IsFourByte ? 4 : 2
)) && "Splat of a value outside of the loaded memory"
) ? void (0) : __assert_fail ("(SplatIdx < (IsFourByte ? 4 : 2)) && \"Splat of a value outside of the loaded memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 9781, __extension__ __PRETTY_FUNCTION__))
;
9782 }
9783
9784 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
9785 // For 4-byte load-and-splat, we need Power9.
9786 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
9787 uint64_t Offset = 0;
9788 if (IsFourByte)
9789 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4;
9790 else
9791 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
9792
9793 // If the width of the load is the same as the width of the splat,
9794 // loading with an offset would load the wrong memory.
9795 if (LD->getValueType(0).getSizeInBits() == (IsFourByte ? 32 : 64))
9796 Offset = 0;
9797
9798 SDValue BasePtr = LD->getBasePtr();
9799 if (Offset != 0)
9800 BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
9801 BasePtr, DAG.getIntPtrConstant(Offset, dl));
9802 SDValue Ops[] = {
9803 LD->getChain(), // Chain
9804 BasePtr, // BasePtr
9805 DAG.getValueType(Op.getValueType()) // VT
9806 };
9807 SDVTList VTL =
9808 DAG.getVTList(IsFourByte ? MVT::v4i32 : MVT::v2i64, MVT::Other);
9809 SDValue LdSplt =
9810 DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, VTL,
9811 Ops, LD->getMemoryVT(), LD->getMemOperand());
9812 DAG.ReplaceAllUsesOfValueWith(InputLoad->getValue(1), LdSplt.getValue(1));
9813 if (LdSplt.getValueType() != SVOp->getValueType(0))
9814 LdSplt = DAG.getBitcast(SVOp->getValueType(0), LdSplt);
9815 return LdSplt;
9816 }
9817 }
9818 if (Subtarget.hasP9Vector() &&
9819 PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap,
9820 isLittleEndian)) {
9821 if (Swap)
9822 std::swap(V1, V2);
9823 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
9824 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2);
9825 if (ShiftElts) {
9826 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2,
9827 DAG.getConstant(ShiftElts, dl, MVT::i32));
9828 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl,
9829 DAG.getConstant(InsertAtByte, dl, MVT::i32));
9830 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
9831 }
9832 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2,
9833 DAG.getConstant(InsertAtByte, dl, MVT::i32));
9834 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins);
9835 }
9836
9837 if (Subtarget.hasPrefixInstrs()) {
9838 SDValue SplatInsertNode;
9839 if ((SplatInsertNode = lowerToXXSPLTI32DX(SVOp, DAG)))
9840 return SplatInsertNode;
9841 }
9842
9843 if (Subtarget.hasP9Altivec()) {
9844 SDValue NewISDNode;
9845 if ((NewISDNode = lowerToVINSERTH(SVOp, DAG)))
9846 return NewISDNode;
9847
9848 if ((NewISDNode = lowerToVINSERTB(SVOp, DAG)))
9849 return NewISDNode;
9850 }
9851
9852 if (Subtarget.hasVSX() &&
9853 PPC::isXXSLDWIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) {
9854 if (Swap)
9855 std::swap(V1, V2);
9856 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
9857 SDValue Conv2 =
9858 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2.isUndef() ? V1 : V2);
9859
9860 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2,
9861 DAG.getConstant(ShiftElts, dl, MVT::i32));
9862 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Shl);
9863 }
9864
9865 if (Subtarget.hasVSX() &&
9866 PPC::isXXPERMDIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) {
9867 if (Swap)
9868 std::swap(V1, V2);
9869 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1);
9870 SDValue Conv2 =
9871 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2.isUndef() ? V1 : V2);
9872
9873 SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2,
9874 DAG.getConstant(ShiftElts, dl, MVT::i32));
9875 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, PermDI);
9876 }
9877
9878 if (Subtarget.hasP9Vector()) {
9879 if (PPC::isXXBRHShuffleMask(SVOp)) {
9880 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
9881 SDValue ReveHWord = DAG.getNode(ISD::BSWAP, dl, MVT::v8i16, Conv);
9882 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveHWord);
9883 } else if (PPC::isXXBRWShuffleMask(SVOp)) {
9884 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
9885 SDValue ReveWord = DAG.getNode(ISD::BSWAP, dl, MVT::v4i32, Conv);
9886 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveWord);
9887 } else if (PPC::isXXBRDShuffleMask(SVOp)) {
9888 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1);
9889 SDValue ReveDWord = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Conv);
9890 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveDWord);
9891 } else if (PPC::isXXBRQShuffleMask(SVOp)) {
9892 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1);
9893 SDValue ReveQWord = DAG.getNode(ISD::BSWAP, dl, MVT::v1i128, Conv);
9894 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveQWord);
9895 }
9896 }
9897
9898 if (Subtarget.hasVSX()) {
9899 if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) {
9900 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, 4, DAG);
9901
9902 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
9903 SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv,
9904 DAG.getConstant(SplatIdx, dl, MVT::i32));
9905 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Splat);
9906 }
9907
9908 // Left shifts of 8 bytes are actually swaps. Convert accordingly.
9909 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) == 8) {
9910 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1);
9911 SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv);
9912 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Swap);
9913 }
9914 }
9915
9916 // Cases that are handled by instructions that take permute immediates
9917 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
9918 // selected by the instruction selector.
9919 if (V2.isUndef()) {
9920 if (PPC::isSplatShuffleMask(SVOp, 1) ||
9921 PPC::isSplatShuffleMask(SVOp, 2) ||
9922 PPC::isSplatShuffleMask(SVOp, 4) ||
9923 PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) ||
9924 PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) ||
9925 PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 ||
9926 PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) ||
9927 PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) ||
9928 PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) ||
9929 PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) ||
9930 PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) ||
9931 PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) ||
9932 (Subtarget.hasP8Altivec() && (
9933 PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) ||
9934 PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) ||
9935 PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) {
9936 return Op;
9937 }
9938 }
9939
9940 // Altivec has a variety of "shuffle immediates" that take two vector inputs
9941 // and produce a fixed permutation. If any of these match, do not lower to
9942 // VPERM.
9943 unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
9944 if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) ||
9945 PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) ||
9946 PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 ||
9947 PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) ||
9948 PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) ||
9949 PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) ||
9950 PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) ||
9951 PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) ||
9952 PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) ||
9953 (Subtarget.hasP8Altivec() && (
9954 PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) ||
9955 PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) ||
9956 PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG))))
9957 return Op;
9958
9959 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
9960 // perfect shuffle table to emit an optimal matching sequence.
9961 ArrayRef<int> PermMask = SVOp->getMask();
9962
9963 unsigned PFIndexes[4];
9964 bool isFourElementShuffle = true;
9965 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number
9966 unsigned EltNo = 8; // Start out undef.
9967 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
9968 if (PermMask[i*4+j] < 0)
9969 continue; // Undef, ignore it.
9970
9971 unsigned ByteSource = PermMask[i*4+j];
9972 if ((ByteSource & 3) != j) {
9973 isFourElementShuffle = false;
9974 break;
9975 }
9976
9977 if (EltNo == 8) {
9978 EltNo = ByteSource/4;
9979 } else if (EltNo != ByteSource/4) {
9980 isFourElementShuffle = false;
9981 break;
9982 }
9983 }
9984 PFIndexes[i] = EltNo;
9985 }
9986
9987 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the
9988 // perfect shuffle vector to determine if it is cost effective to do this as
9989 // discrete instructions, or whether we should use a vperm.
9990 // For now, we skip this for little endian until such time as we have a
9991 // little-endian perfect shuffle table.
9992 if (isFourElementShuffle && !isLittleEndian) {
9993 // Compute the index in the perfect shuffle table.
9994 unsigned PFTableIndex =
9995 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
9996
9997 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
9998 unsigned Cost = (PFEntry >> 30);
9999
10000 // Determining when to avoid vperm is tricky. Many things affect the cost
10001 // of vperm, particularly how many times the perm mask needs to be computed.
10002 // For example, if the perm mask can be hoisted out of a loop or is already
10003 // used (perhaps because there are multiple permutes with the same shuffle
10004 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of
10005 // the loop requires an extra register.
10006 //
10007 // As a compromise, we only emit discrete instructions if the shuffle can be
10008 // generated in 3 or fewer operations. When we have loop information
10009 // available, if this block is within a loop, we should avoid using vperm
10010 // for 3-operation perms and use a constant pool load instead.
10011 if (Cost < 3)
10012 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
10013 }
10014
10015 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant
10016 // vector that will get spilled to the constant pool.
10017 if (V2.isUndef()) V2 = V1;
10018
10019 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except
10020 // that it is in input element units, not in bytes. Convert now.
10021
10022 // For little endian, the order of the input vectors is reversed, and
10023 // the permutation mask is complemented with respect to 31. This is
10024 // necessary to produce proper semantics with the big-endian-biased vperm
10025 // instruction.
10026 EVT EltVT = V1.getValueType().getVectorElementType();
10027 unsigned BytesPerElement = EltVT.getSizeInBits()/8;
10028
10029 SmallVector<SDValue, 16> ResultMask;
10030 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
10031 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
10032
10033 for (unsigned j = 0; j != BytesPerElement; ++j)
10034 if (isLittleEndian)
10035 ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j),
10036 dl, MVT::i32));
10037 else
10038 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl,
10039 MVT::i32));
10040 }
10041
10042 ShufflesHandledWithVPERM++;
10043 SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask);
10044 LLVM_DEBUG(dbgs() << "Emitting a VPERM for the following shuffle:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { dbgs() << "Emitting a VPERM for the following shuffle:\n"
; } } while (false)
;
10045 LLVM_DEBUG(SVOp->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { SVOp->dump(); } } while (false)
;
10046 LLVM_DEBUG(dbgs() << "With the following permute control vector:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { dbgs() << "With the following permute control vector:\n"
; } } while (false)
;
10047 LLVM_DEBUG(VPermMask.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-lowering")) { VPermMask.dump(); } } while (false)
;
10048
10049 if (isLittleEndian)
10050 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(),
10051 V2, V1, VPermMask);
10052 else
10053 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(),
10054 V1, V2, VPermMask);
10055}
10056
10057/// getVectorCompareInfo - Given an intrinsic, return false if it is not a
10058/// vector comparison. If it is, return true and fill in Opc/isDot with
10059/// information about the intrinsic.
10060static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc,
10061 bool &isDot, const PPCSubtarget &Subtarget) {
10062 unsigned IntrinsicID =
10063 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue();
10064 CompareOpc = -1;
10065 isDot = false;
10066 switch (IntrinsicID) {
10067 default:
10068 return false;
10069 // Comparison predicates.
10070 case Intrinsic::ppc_altivec_vcmpbfp_p:
10071 CompareOpc = 966;
10072 isDot = true;
10073 break;
10074 case Intrinsic::ppc_altivec_vcmpeqfp_p:
10075 CompareOpc = 198;
10076 isDot = true;
10077 break;
10078 case Intrinsic::ppc_altivec_vcmpequb_p:
10079 CompareOpc = 6;
10080 isDot = true;
10081 break;
10082 case Intrinsic::ppc_altivec_vcmpequh_p:
10083 CompareOpc = 70;
10084 isDot = true;
10085 break;
10086 case Intrinsic::ppc_altivec_vcmpequw_p:
10087 CompareOpc = 134;
10088 isDot = true;
10089 break;
10090 case Intrinsic::ppc_altivec_vcmpequd_p:
10091 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10092 CompareOpc = 199;
10093 isDot = true;
10094 } else
10095 return false;
10096 break;
10097 case Intrinsic::ppc_altivec_vcmpneb_p:
10098 case Intrinsic::ppc_altivec_vcmpneh_p:
10099 case Intrinsic::ppc_altivec_vcmpnew_p:
10100 case Intrinsic::ppc_altivec_vcmpnezb_p:
10101 case Intrinsic::ppc_altivec_vcmpnezh_p:
10102 case Intrinsic::ppc_altivec_vcmpnezw_p:
10103 if (Subtarget.hasP9Altivec()) {
10104 switch (IntrinsicID) {
10105 default:
10106 llvm_unreachable("Unknown comparison intrinsic.")::llvm::llvm_unreachable_internal("Unknown comparison intrinsic."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10106)
;
10107 case Intrinsic::ppc_altivec_vcmpneb_p:
10108 CompareOpc = 7;
10109 break;
10110 case Intrinsic::ppc_altivec_vcmpneh_p:
10111 CompareOpc = 71;
10112 break;
10113 case Intrinsic::ppc_altivec_vcmpnew_p:
10114 CompareOpc = 135;
10115 break;
10116 case Intrinsic::ppc_altivec_vcmpnezb_p:
10117 CompareOpc = 263;
10118 break;
10119 case Intrinsic::ppc_altivec_vcmpnezh_p:
10120 CompareOpc = 327;
10121 break;
10122 case Intrinsic::ppc_altivec_vcmpnezw_p:
10123 CompareOpc = 391;
10124 break;
10125 }
10126 isDot = true;
10127 } else
10128 return false;
10129 break;
10130 case Intrinsic::ppc_altivec_vcmpgefp_p:
10131 CompareOpc = 454;
10132 isDot = true;
10133 break;
10134 case Intrinsic::ppc_altivec_vcmpgtfp_p:
10135 CompareOpc = 710;
10136 isDot = true;
10137 break;
10138 case Intrinsic::ppc_altivec_vcmpgtsb_p:
10139 CompareOpc = 774;
10140 isDot = true;
10141 break;
10142 case Intrinsic::ppc_altivec_vcmpgtsh_p:
10143 CompareOpc = 838;
10144 isDot = true;
10145 break;
10146 case Intrinsic::ppc_altivec_vcmpgtsw_p:
10147 CompareOpc = 902;
10148 isDot = true;
10149 break;
10150 case Intrinsic::ppc_altivec_vcmpgtsd_p:
10151 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10152 CompareOpc = 967;
10153 isDot = true;
10154 } else
10155 return false;
10156 break;
10157 case Intrinsic::ppc_altivec_vcmpgtub_p:
10158 CompareOpc = 518;
10159 isDot = true;
10160 break;
10161 case Intrinsic::ppc_altivec_vcmpgtuh_p:
10162 CompareOpc = 582;
10163 isDot = true;
10164 break;
10165 case Intrinsic::ppc_altivec_vcmpgtuw_p:
10166 CompareOpc = 646;
10167 isDot = true;
10168 break;
10169 case Intrinsic::ppc_altivec_vcmpgtud_p:
10170 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10171 CompareOpc = 711;
10172 isDot = true;
10173 } else
10174 return false;
10175 break;
10176
10177 case Intrinsic::ppc_altivec_vcmpequq:
10178 case Intrinsic::ppc_altivec_vcmpgtsq:
10179 case Intrinsic::ppc_altivec_vcmpgtuq:
10180 if (!Subtarget.isISA3_1())
10181 return false;
10182 switch (IntrinsicID) {
10183 default:
10184 llvm_unreachable("Unknown comparison intrinsic.")::llvm::llvm_unreachable_internal("Unknown comparison intrinsic."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10184)
;
10185 case Intrinsic::ppc_altivec_vcmpequq:
10186 CompareOpc = 455;
10187 break;
10188 case Intrinsic::ppc_altivec_vcmpgtsq:
10189 CompareOpc = 903;
10190 break;
10191 case Intrinsic::ppc_altivec_vcmpgtuq:
10192 CompareOpc = 647;
10193 break;
10194 }
10195 break;
10196
10197 // VSX predicate comparisons use the same infrastructure
10198 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10199 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10200 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
10201 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
10202 case Intrinsic::ppc_vsx_xvcmpgesp_p:
10203 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
10204 if (Subtarget.hasVSX()) {
10205 switch (IntrinsicID) {
10206 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10207 CompareOpc = 99;
10208 break;
10209 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10210 CompareOpc = 115;
10211 break;
10212 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
10213 CompareOpc = 107;
10214 break;
10215 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
10216 CompareOpc = 67;
10217 break;
10218 case Intrinsic::ppc_vsx_xvcmpgesp_p:
10219 CompareOpc = 83;
10220 break;
10221 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
10222 CompareOpc = 75;
10223 break;
10224 }
10225 isDot = true;
10226 } else
10227 return false;
10228 break;
10229
10230 // Normal Comparisons.
10231 case Intrinsic::ppc_altivec_vcmpbfp:
10232 CompareOpc = 966;
10233 break;
10234 case Intrinsic::ppc_altivec_vcmpeqfp:
10235 CompareOpc = 198;
10236 break;
10237 case Intrinsic::ppc_altivec_vcmpequb:
10238 CompareOpc = 6;
10239 break;
10240 case Intrinsic::ppc_altivec_vcmpequh:
10241 CompareOpc = 70;
10242 break;
10243 case Intrinsic::ppc_altivec_vcmpequw:
10244 CompareOpc = 134;
10245 break;
10246 case Intrinsic::ppc_altivec_vcmpequd:
10247 if (Subtarget.hasP8Altivec())
10248 CompareOpc = 199;
10249 else
10250 return false;
10251 break;
10252 case Intrinsic::ppc_altivec_vcmpneb:
10253 case Intrinsic::ppc_altivec_vcmpneh:
10254 case Intrinsic::ppc_altivec_vcmpnew:
10255 case Intrinsic::ppc_altivec_vcmpnezb:
10256 case Intrinsic::ppc_altivec_vcmpnezh:
10257 case Intrinsic::ppc_altivec_vcmpnezw:
10258 if (Subtarget.hasP9Altivec())
10259 switch (IntrinsicID) {
10260 default:
10261 llvm_unreachable("Unknown comparison intrinsic.")::llvm::llvm_unreachable_internal("Unknown comparison intrinsic."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10261)
;
10262 case Intrinsic::ppc_altivec_vcmpneb:
10263 CompareOpc = 7;
10264 break;
10265 case Intrinsic::ppc_altivec_vcmpneh:
10266 CompareOpc = 71;
10267 break;
10268 case Intrinsic::ppc_altivec_vcmpnew:
10269 CompareOpc = 135;
10270 break;
10271 case Intrinsic::ppc_altivec_vcmpnezb:
10272 CompareOpc = 263;
10273 break;
10274 case Intrinsic::ppc_altivec_vcmpnezh:
10275 CompareOpc = 327;
10276 break;
10277 case Intrinsic::ppc_altivec_vcmpnezw:
10278 CompareOpc = 391;
10279 break;
10280 }
10281 else
10282 return false;
10283 break;
10284 case Intrinsic::ppc_altivec_vcmpgefp:
10285 CompareOpc = 454;
10286 break;
10287 case Intrinsic::ppc_altivec_vcmpgtfp:
10288 CompareOpc = 710;
10289 break;
10290 case Intrinsic::ppc_altivec_vcmpgtsb:
10291 CompareOpc = 774;
10292 break;
10293 case Intrinsic::ppc_altivec_vcmpgtsh:
10294 CompareOpc = 838;
10295 break;
10296 case Intrinsic::ppc_altivec_vcmpgtsw:
10297 CompareOpc = 902;
10298 break;
10299 case Intrinsic::ppc_altivec_vcmpgtsd:
10300 if (Subtarget.hasP8Altivec())
10301 CompareOpc = 967;
10302 else
10303 return false;
10304 break;
10305 case Intrinsic::ppc_altivec_vcmpgtub:
10306 CompareOpc = 518;
10307 break;
10308 case Intrinsic::ppc_altivec_vcmpgtuh:
10309 CompareOpc = 582;
10310 break;
10311 case Intrinsic::ppc_altivec_vcmpgtuw:
10312 CompareOpc = 646;
10313 break;
10314 case Intrinsic::ppc_altivec_vcmpgtud:
10315 if (Subtarget.hasP8Altivec())
10316 CompareOpc = 711;
10317 else
10318 return false;
10319 break;
10320 case Intrinsic::ppc_altivec_vcmpequq_p:
10321 case Intrinsic::ppc_altivec_vcmpgtsq_p:
10322 case Intrinsic::ppc_altivec_vcmpgtuq_p:
10323 if (!Subtarget.isISA3_1())
10324 return false;
10325 switch (IntrinsicID) {
10326 default:
10327 llvm_unreachable("Unknown comparison intrinsic.")::llvm::llvm_unreachable_internal("Unknown comparison intrinsic."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10327)
;
10328 case Intrinsic::ppc_altivec_vcmpequq_p:
10329 CompareOpc = 455;
10330 break;
10331 case Intrinsic::ppc_altivec_vcmpgtsq_p:
10332 CompareOpc = 903;
10333 break;
10334 case Intrinsic::ppc_altivec_vcmpgtuq_p:
10335 CompareOpc = 647;
10336 break;
10337 }
10338 isDot = true;
10339 break;
10340 }
10341 return true;
10342}
10343
10344/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom
10345/// lower, do it, otherwise return null.
10346SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
10347 SelectionDAG &DAG) const {
10348 unsigned IntrinsicID =
10349 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
10350
10351 SDLoc dl(Op);
10352
10353 switch (IntrinsicID) {
10354 case Intrinsic::thread_pointer:
10355 // Reads the thread pointer register, used for __builtin_thread_pointer.
10356 if (Subtarget.isPPC64())
10357 return DAG.getRegister(PPC::X13, MVT::i64);
10358 return DAG.getRegister(PPC::R2, MVT::i32);
10359
10360 case Intrinsic::ppc_mma_disassemble_acc:
10361 case Intrinsic::ppc_vsx_disassemble_pair: {
10362 int NumVecs = 2;
10363 SDValue WideVec = Op.getOperand(1);
10364 if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) {
10365 NumVecs = 4;
10366 WideVec = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, WideVec);
10367 }
10368 SmallVector<SDValue, 4> RetOps;
10369 for (int VecNo = 0; VecNo < NumVecs; VecNo++) {
10370 SDValue Extract = DAG.getNode(
10371 PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, WideVec,
10372 DAG.getConstant(Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
10373 : VecNo,
10374 dl, getPointerTy(DAG.getDataLayout())));
10375 RetOps.push_back(Extract);
10376 }
10377 return DAG.getMergeValues(RetOps, dl);
10378 }
10379 }
10380
10381 // If this is a lowered altivec predicate compare, CompareOpc is set to the
10382 // opcode number of the comparison.
10383 int CompareOpc;
10384 bool isDot;
10385 if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget))
10386 return SDValue(); // Don't custom lower most intrinsics.
10387
10388 // If this is a non-dot comparison, make the VCMP node and we are done.
10389 if (!isDot) {
10390 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(),
10391 Op.getOperand(1), Op.getOperand(2),
10392 DAG.getConstant(CompareOpc, dl, MVT::i32));
10393 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp);
10394 }
10395
10396 // Create the PPCISD altivec 'dot' comparison node.
10397 SDValue Ops[] = {
10398 Op.getOperand(2), // LHS
10399 Op.getOperand(3), // RHS
10400 DAG.getConstant(CompareOpc, dl, MVT::i32)
10401 };
10402 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue };
10403 SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops);
10404
10405 // Now that we have the comparison, emit a copy from the CR to a GPR.
10406 // This is flagged to the above dot comparison.
10407 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32,
10408 DAG.getRegister(PPC::CR6, MVT::i32),
10409 CompNode.getValue(1));
10410
10411 // Unpack the result based on how the target uses it.
10412 unsigned BitNo; // Bit # of CR6.
10413 bool InvertBit; // Invert result?
10414 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) {
10415 default: // Can't happen, don't crash on invalid number though.
10416 case 0: // Return the value of the EQ bit of CR6.
10417 BitNo = 0; InvertBit = false;
10418 break;
10419 case 1: // Return the inverted value of the EQ bit of CR6.
10420 BitNo = 0; InvertBit = true;
10421 break;
10422 case 2: // Return the value of the LT bit of CR6.
10423 BitNo = 2; InvertBit = false;
10424 break;
10425 case 3: // Return the inverted value of the LT bit of CR6.
10426 BitNo = 2; InvertBit = true;
10427 break;
10428 }
10429
10430 // Shift the bit into the low position.
10431 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags,
10432 DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32));
10433 // Isolate the bit.
10434 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags,
10435 DAG.getConstant(1, dl, MVT::i32));
10436
10437 // If we are supposed to, toggle the bit.
10438 if (InvertBit)
10439 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags,
10440 DAG.getConstant(1, dl, MVT::i32));
10441 return Flags;
10442}
10443
10444SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
10445 SelectionDAG &DAG) const {
10446 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to
10447 // the beginning of the argument list.
10448 int ArgStart = isa<ConstantSDNode>(Op.getOperand(0)) ? 0 : 1;
10449 SDLoc DL(Op);
10450 switch (cast<ConstantSDNode>(Op.getOperand(ArgStart))->getZExtValue()) {
10451 case Intrinsic::ppc_cfence: {
10452 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument.")(static_cast <bool> (ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument."
) ? void (0) : __assert_fail ("ArgStart == 1 && \"llvm.ppc.cfence must carry a chain argument.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10452, __extension__ __PRETTY_FUNCTION__))
;
10453 assert(Subtarget.isPPC64() && "Only 64-bit is supported for now.")(static_cast <bool> (Subtarget.isPPC64() && "Only 64-bit is supported for now."
) ? void (0) : __assert_fail ("Subtarget.isPPC64() && \"Only 64-bit is supported for now.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10453, __extension__ __PRETTY_FUNCTION__))
;
10454 return SDValue(DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other,
10455 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,
10456 Op.getOperand(ArgStart + 1)),
10457 Op.getOperand(0)),
10458 0);
10459 }
10460 default:
10461 break;
10462 }
10463 return SDValue();
10464}
10465
10466// Lower scalar BSWAP64 to xxbrd.
10467SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
10468 SDLoc dl(Op);
10469 if (!Subtarget.isPPC64())
10470 return Op;
10471 // MTVSRDD
10472 Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0),
10473 Op.getOperand(0));
10474 // XXBRD
10475 Op = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Op);
10476 // MFVSRD
10477 int VectorIndex = 0;
10478 if (Subtarget.isLittleEndian())
10479 VectorIndex = 1;
10480 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op,
10481 DAG.getTargetConstant(VectorIndex, dl, MVT::i32));
10482 return Op;
10483}
10484
10485// ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be
10486// compared to a value that is atomically loaded (atomic loads zero-extend).
10487SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
10488 SelectionDAG &DAG) const {
10489 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP &&(static_cast <bool> (Op.getOpcode() == ISD::ATOMIC_CMP_SWAP
&& "Expecting an atomic compare-and-swap here.") ? void
(0) : __assert_fail ("Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && \"Expecting an atomic compare-and-swap here.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10490, __extension__ __PRETTY_FUNCTION__))
10490 "Expecting an atomic compare-and-swap here.")(static_cast <bool> (Op.getOpcode() == ISD::ATOMIC_CMP_SWAP
&& "Expecting an atomic compare-and-swap here.") ? void
(0) : __assert_fail ("Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && \"Expecting an atomic compare-and-swap here.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10490, __extension__ __PRETTY_FUNCTION__))
;
10491 SDLoc dl(Op);
10492 auto *AtomicNode = cast<AtomicSDNode>(Op.getNode());
10493 EVT MemVT = AtomicNode->getMemoryVT();
10494 if (MemVT.getSizeInBits() >= 32)
10495 return Op;
10496
10497 SDValue CmpOp = Op.getOperand(2);
10498 // If this is already correctly zero-extended, leave it alone.
10499 auto HighBits = APInt::getHighBitsSet(32, 32 - MemVT.getSizeInBits());
10500 if (DAG.MaskedValueIsZero(CmpOp, HighBits))
10501 return Op;
10502
10503 // Clear the high bits of the compare operand.
10504 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1;
10505 SDValue NewCmpOp =
10506 DAG.getNode(ISD::AND, dl, MVT::i32, CmpOp,
10507 DAG.getConstant(MaskVal, dl, MVT::i32));
10508
10509 // Replace the existing compare operand with the properly zero-extended one.
10510 SmallVector<SDValue, 4> Ops;
10511 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++)
10512 Ops.push_back(AtomicNode->getOperand(i));
10513 Ops[2] = NewCmpOp;
10514 MachineMemOperand *MMO = AtomicNode->getMemOperand();
10515 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other);
10516 auto NodeTy =
10517 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16;
10518 return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO);
10519}
10520
10521SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
10522 SelectionDAG &DAG) const {
10523 SDLoc dl(Op);
10524 // Create a stack slot that is 16-byte aligned.
10525 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10526 int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
10527 EVT PtrVT = getPointerTy(DAG.getDataLayout());
10528 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
10529
10530 // Store the input value into Value#0 of the stack slot.
10531 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx,
10532 MachinePointerInfo());
10533 // Load it out.
10534 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo());
10535}
10536
10537SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
10538 SelectionDAG &DAG) const {
10539 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&(static_cast <bool> (Op.getOpcode() == ISD::INSERT_VECTOR_ELT
&& "Should only be called for ISD::INSERT_VECTOR_ELT"
) ? void (0) : __assert_fail ("Op.getOpcode() == ISD::INSERT_VECTOR_ELT && \"Should only be called for ISD::INSERT_VECTOR_ELT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10540, __extension__ __PRETTY_FUNCTION__))
10540 "Should only be called for ISD::INSERT_VECTOR_ELT")(static_cast <bool> (Op.getOpcode() == ISD::INSERT_VECTOR_ELT
&& "Should only be called for ISD::INSERT_VECTOR_ELT"
) ? void (0) : __assert_fail ("Op.getOpcode() == ISD::INSERT_VECTOR_ELT && \"Should only be called for ISD::INSERT_VECTOR_ELT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10540, __extension__ __PRETTY_FUNCTION__))
;
10541
10542 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(2));
10543
10544 EVT VT = Op.getValueType();
10545 SDLoc dl(Op);
10546 SDValue V1 = Op.getOperand(0);
10547 SDValue V2 = Op.getOperand(1);
10548 SDValue V3 = Op.getOperand(2);
10549
10550 if (VT == MVT::v2f64 && C)
10551 return Op;
10552
10553 if (Subtarget.isISA3_1()) {
10554 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64())
10555 return SDValue();
10556 // On P10, we have legal lowering for constant and variable indices for
10557 // integer vectors.
10558 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
10559 VT == MVT::v2i64)
10560 return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, V2, V3);
10561 // For f32 and f64 vectors, we have legal lowering for variable indices.
10562 // For f32 we also have legal lowering when the element is loaded from
10563 // memory.
10564 if (VT == MVT::v4f32 || VT == MVT::v2f64) {
10565 if (!C || (VT == MVT::v4f32 && dyn_cast<LoadSDNode>(V2)))
10566 return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, V2, V3);
10567 return Op;
10568 }
10569 }
10570
10571 // Before P10, we have legal lowering for constant indices but not for
10572 // variable ones.
10573 if (!C)
10574 return SDValue();
10575
10576 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types.
10577 if (VT == MVT::v8i16 || VT == MVT::v16i8) {
10578 SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2);
10579 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8;
10580 unsigned InsertAtElement = C->getZExtValue();
10581 unsigned InsertAtByte = InsertAtElement * BytesInEachElement;
10582 if (Subtarget.isLittleEndian()) {
10583 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte;
10584 }
10585 return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, Mtvsrz,
10586 DAG.getConstant(InsertAtByte, dl, MVT::i32));
10587 }
10588 return Op;
10589}
10590
10591SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
10592 SelectionDAG &DAG) const {
10593 SDLoc dl(Op);
10594 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
10595 SDValue LoadChain = LN->getChain();
10596 SDValue BasePtr = LN->getBasePtr();
10597 EVT VT = Op.getValueType();
10598
10599 if (VT != MVT::v256i1 && VT != MVT::v512i1)
10600 return Op;
10601
10602 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
10603 // Here we create 2 or 4 v16i8 loads to load the pair or accumulator value in
10604 // 2 or 4 vsx registers.
10605 assert((VT != MVT::v512i1 || Subtarget.hasMMA()) &&(static_cast <bool> ((VT != MVT::v512i1 || Subtarget.hasMMA
()) && "Type unsupported without MMA") ? void (0) : __assert_fail
("(VT != MVT::v512i1 || Subtarget.hasMMA()) && \"Type unsupported without MMA\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10606, __extension__ __PRETTY_FUNCTION__))
10606 "Type unsupported without MMA")(static_cast <bool> ((VT != MVT::v512i1 || Subtarget.hasMMA
()) && "Type unsupported without MMA") ? void (0) : __assert_fail
("(VT != MVT::v512i1 || Subtarget.hasMMA()) && \"Type unsupported without MMA\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10606, __extension__ __PRETTY_FUNCTION__))
;
10607 assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&(static_cast <bool> ((VT != MVT::v256i1 || Subtarget.pairedVectorMemops
()) && "Type unsupported without paired vector support"
) ? void (0) : __assert_fail ("(VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && \"Type unsupported without paired vector support\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10608, __extension__ __PRETTY_FUNCTION__))
10608 "Type unsupported without paired vector support")(static_cast <bool> ((VT != MVT::v256i1 || Subtarget.pairedVectorMemops
()) && "Type unsupported without paired vector support"
) ? void (0) : __assert_fail ("(VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && \"Type unsupported without paired vector support\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10608, __extension__ __PRETTY_FUNCTION__))
;
10609 Align Alignment = LN->getAlign();
10610 SmallVector<SDValue, 4> Loads;
10611 SmallVector<SDValue, 4> LoadChains;
10612 unsigned NumVecs = VT.getSizeInBits() / 128;
10613 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
10614 SDValue Load =
10615 DAG.getLoad(MVT::v16i8, dl, LoadChain, BasePtr,
10616 LN->getPointerInfo().getWithOffset(Idx * 16),
10617 commonAlignment(Alignment, Idx * 16),
10618 LN->getMemOperand()->getFlags(), LN->getAAInfo());
10619 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
10620 DAG.getConstant(16, dl, BasePtr.getValueType()));
10621 Loads.push_back(Load);
10622 LoadChains.push_back(Load.getValue(1));
10623 }
10624 if (Subtarget.isLittleEndian()) {
10625 std::reverse(Loads.begin(), Loads.end());
10626 std::reverse(LoadChains.begin(), LoadChains.end());
10627 }
10628 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
10629 SDValue Value =
10630 DAG.getNode(VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD,
10631 dl, VT, Loads);
10632 SDValue RetOps[] = {Value, TF};
10633 return DAG.getMergeValues(RetOps, dl);
10634}
10635
10636SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
10637 SelectionDAG &DAG) const {
10638 SDLoc dl(Op);
10639 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
10640 SDValue StoreChain = SN->getChain();
10641 SDValue BasePtr = SN->getBasePtr();
10642 SDValue Value = SN->getValue();
10643 EVT StoreVT = Value.getValueType();
10644
10645 if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1)
10646 return Op;
10647
10648 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
10649 // Here we create 2 or 4 v16i8 stores to store the pair or accumulator
10650 // underlying registers individually.
10651 assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) &&(static_cast <bool> ((StoreVT != MVT::v512i1 || Subtarget
.hasMMA()) && "Type unsupported without MMA") ? void (
0) : __assert_fail ("(StoreVT != MVT::v512i1 || Subtarget.hasMMA()) && \"Type unsupported without MMA\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10652, __extension__ __PRETTY_FUNCTION__))
10652 "Type unsupported without MMA")(static_cast <bool> ((StoreVT != MVT::v512i1 || Subtarget
.hasMMA()) && "Type unsupported without MMA") ? void (
0) : __assert_fail ("(StoreVT != MVT::v512i1 || Subtarget.hasMMA()) && \"Type unsupported without MMA\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10652, __extension__ __PRETTY_FUNCTION__))
;
10653 assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&(static_cast <bool> ((StoreVT != MVT::v256i1 || Subtarget
.pairedVectorMemops()) && "Type unsupported without paired vector support"
) ? void (0) : __assert_fail ("(StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && \"Type unsupported without paired vector support\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10654, __extension__ __PRETTY_FUNCTION__))
10654 "Type unsupported without paired vector support")(static_cast <bool> ((StoreVT != MVT::v256i1 || Subtarget
.pairedVectorMemops()) && "Type unsupported without paired vector support"
) ? void (0) : __assert_fail ("(StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && \"Type unsupported without paired vector support\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10654, __extension__ __PRETTY_FUNCTION__))
;
10655 Align Alignment = SN->getAlign();
10656 SmallVector<SDValue, 4> Stores;
10657 unsigned NumVecs = 2;
10658 if (StoreVT == MVT::v512i1) {
10659 Value = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, Value);
10660 NumVecs = 4;
10661 }
10662 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
10663 unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx;
10664 SDValue Elt = DAG.getNode(PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, Value,
10665 DAG.getConstant(VecNum, dl, getPointerTy(DAG.getDataLayout())));
10666 SDValue Store =
10667 DAG.getStore(StoreChain, dl, Elt, BasePtr,
10668 SN->getPointerInfo().getWithOffset(Idx * 16),
10669 commonAlignment(Alignment, Idx * 16),
10670 SN->getMemOperand()->getFlags(), SN->getAAInfo());
10671 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
10672 DAG.getConstant(16, dl, BasePtr.getValueType()));
10673 Stores.push_back(Store);
10674 }
10675 SDValue TF = DAG.getTokenFactor(dl, Stores);
10676 return TF;
10677}
10678
10679SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
10680 SDLoc dl(Op);
10681 if (Op.getValueType() == MVT::v4i32) {
10682 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
10683
10684 SDValue Zero = getCanonicalConstSplat(0, 1, MVT::v4i32, DAG, dl);
10685 // +16 as shift amt.
10686 SDValue Neg16 = getCanonicalConstSplat(-16, 4, MVT::v4i32, DAG, dl);
10687 SDValue RHSSwap = // = vrlw RHS, 16
10688 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl);
10689
10690 // Shrinkify inputs to v8i16.
10691 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS);
10692 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS);
10693 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap);
10694
10695 // Low parts multiplied together, generating 32-bit results (we ignore the
10696 // top parts).
10697 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh,
10698 LHS, RHS, DAG, dl, MVT::v4i32);
10699
10700 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm,
10701 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32);
10702 // Shift the high parts up 16 bits.
10703 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd,
10704 Neg16, DAG, dl);
10705 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd);
10706 } else if (Op.getValueType() == MVT::v16i8) {
10707 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
10708 bool isLittleEndian = Subtarget.isLittleEndian();
10709
10710 // Multiply the even 8-bit parts, producing 16-bit sums.
10711 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub,
10712 LHS, RHS, DAG, dl, MVT::v8i16);
10713 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts);
10714
10715 // Multiply the odd 8-bit parts, producing 16-bit sums.
10716 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub,
10717 LHS, RHS, DAG, dl, MVT::v8i16);
10718 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts);
10719
10720 // Merge the results together. Because vmuleub and vmuloub are
10721 // instructions with a big-endian bias, we must reverse the
10722 // element numbering and reverse the meaning of "odd" and "even"
10723 // when generating little endian code.
10724 int Ops[16];
10725 for (unsigned i = 0; i != 8; ++i) {
10726 if (isLittleEndian) {
10727 Ops[i*2 ] = 2*i;
10728 Ops[i*2+1] = 2*i+16;
10729 } else {
10730 Ops[i*2 ] = 2*i+1;
10731 Ops[i*2+1] = 2*i+1+16;
10732 }
10733 }
10734 if (isLittleEndian)
10735 return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops);
10736 else
10737 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops);
10738 } else {
10739 llvm_unreachable("Unknown mul to lower!")::llvm::llvm_unreachable_internal("Unknown mul to lower!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10739)
;
10740 }
10741}
10742
10743SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
10744 bool IsStrict = Op->isStrictFPOpcode();
10745 if (Op.getOperand(IsStrict ? 1 : 0).getValueType() == MVT::f128 &&
10746 !Subtarget.hasP9Vector())
10747 return SDValue();
10748
10749 return Op;
10750}
10751
10752// Custom lowering for fpext vf32 to v2f64
10753SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
10754
10755 assert(Op.getOpcode() == ISD::FP_EXTEND &&(static_cast <bool> (Op.getOpcode() == ISD::FP_EXTEND &&
"Should only be called for ISD::FP_EXTEND") ? void (0) : __assert_fail
("Op.getOpcode() == ISD::FP_EXTEND && \"Should only be called for ISD::FP_EXTEND\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10756, __extension__ __PRETTY_FUNCTION__))
10756 "Should only be called for ISD::FP_EXTEND")(static_cast <bool> (Op.getOpcode() == ISD::FP_EXTEND &&
"Should only be called for ISD::FP_EXTEND") ? void (0) : __assert_fail
("Op.getOpcode() == ISD::FP_EXTEND && \"Should only be called for ISD::FP_EXTEND\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10756, __extension__ __PRETTY_FUNCTION__))
;
10757
10758 // FIXME: handle extends from half precision float vectors on P9.
10759 // We only want to custom lower an extend from v2f32 to v2f64.
10760 if (Op.getValueType() != MVT::v2f64 ||
10761 Op.getOperand(0).getValueType() != MVT::v2f32)
10762 return SDValue();
10763
10764 SDLoc dl(Op);
10765 SDValue Op0 = Op.getOperand(0);
10766
10767 switch (Op0.getOpcode()) {
10768 default:
10769 return SDValue();
10770 case ISD::EXTRACT_SUBVECTOR: {
10771 assert(Op0.getNumOperands() == 2 &&(static_cast <bool> (Op0.getNumOperands() == 2 &&
isa<ConstantSDNode>(Op0->getOperand(1)) && "Node should have 2 operands with second one being a constant!"
) ? void (0) : __assert_fail ("Op0.getNumOperands() == 2 && isa<ConstantSDNode>(Op0->getOperand(1)) && \"Node should have 2 operands with second one being a constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10773, __extension__ __PRETTY_FUNCTION__))
10772 isa<ConstantSDNode>(Op0->getOperand(1)) &&(static_cast <bool> (Op0.getNumOperands() == 2 &&
isa<ConstantSDNode>(Op0->getOperand(1)) && "Node should have 2 operands with second one being a constant!"
) ? void (0) : __assert_fail ("Op0.getNumOperands() == 2 && isa<ConstantSDNode>(Op0->getOperand(1)) && \"Node should have 2 operands with second one being a constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10773, __extension__ __PRETTY_FUNCTION__))
10773 "Node should have 2 operands with second one being a constant!")(static_cast <bool> (Op0.getNumOperands() == 2 &&
isa<ConstantSDNode>(Op0->getOperand(1)) && "Node should have 2 operands with second one being a constant!"
) ? void (0) : __assert_fail ("Op0.getNumOperands() == 2 && isa<ConstantSDNode>(Op0->getOperand(1)) && \"Node should have 2 operands with second one being a constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10773, __extension__ __PRETTY_FUNCTION__))
;
10774
10775 if (Op0.getOperand(0).getValueType() != MVT::v4f32)
10776 return SDValue();
10777
10778 // Custom lower is only done for high or low doubleword.
10779 int Idx = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
10780 if (Idx % 2 != 0)
10781 return SDValue();
10782
10783 // Since input is v4f32, at this point Idx is either 0 or 2.
10784 // Shift to get the doubleword position we want.
10785 int DWord = Idx >> 1;
10786
10787 // High and low word positions are different on little endian.
10788 if (Subtarget.isLittleEndian())
10789 DWord ^= 0x1;
10790
10791 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64,
10792 Op0.getOperand(0), DAG.getConstant(DWord, dl, MVT::i32));
10793 }
10794 case ISD::FADD:
10795 case ISD::FMUL:
10796 case ISD::FSUB: {
10797 SDValue NewLoad[2];
10798 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) {
10799 // Ensure both input are loads.
10800 SDValue LdOp = Op0.getOperand(i);
10801 if (LdOp.getOpcode() != ISD::LOAD)
10802 return SDValue();
10803 // Generate new load node.
10804 LoadSDNode *LD = cast<LoadSDNode>(LdOp);
10805 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
10806 NewLoad[i] = DAG.getMemIntrinsicNode(
10807 PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps,
10808 LD->getMemoryVT(), LD->getMemOperand());
10809 }
10810 SDValue NewOp =
10811 DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, NewLoad[0],
10812 NewLoad[1], Op0.getNode()->getFlags());
10813 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewOp,
10814 DAG.getConstant(0, dl, MVT::i32));
10815 }
10816 case ISD::LOAD: {
10817 LoadSDNode *LD = cast<LoadSDNode>(Op0);
10818 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
10819 SDValue NewLd = DAG.getMemIntrinsicNode(
10820 PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps,
10821 LD->getMemoryVT(), LD->getMemOperand());
10822 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewLd,
10823 DAG.getConstant(0, dl, MVT::i32));
10824 }
10825 }
10826 llvm_unreachable("ERROR:Should return for all cases within swtich.")::llvm::llvm_unreachable_internal("ERROR:Should return for all cases within swtich."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10826)
;
10827}
10828
10829/// LowerOperation - Provide custom lowering hooks for some operations.
10830///
10831SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10832 switch (Op.getOpcode()) {
10833 default: llvm_unreachable("Wasn't expecting to be able to lower this!")::llvm::llvm_unreachable_internal("Wasn't expecting to be able to lower this!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10833)
;
10834 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10835 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10836 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10837 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10838 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
10839 case ISD::STRICT_FSETCC:
10840 case ISD::STRICT_FSETCCS:
10841 case ISD::SETCC: return LowerSETCC(Op, DAG);
10842 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
10843 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
10844
10845 case ISD::INLINEASM:
10846 case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG);
10847 // Variable argument lowering.
10848 case ISD::VASTART: return LowerVASTART(Op, DAG);
10849 case ISD::VAARG: return LowerVAARG(Op, DAG);
10850 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
10851
10852 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG);
10853 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
10854 case ISD::GET_DYNAMIC_AREA_OFFSET:
10855 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
10856
10857 // Exception handling lowering.
10858 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG);
10859 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
10860 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
10861
10862 case ISD::LOAD: return LowerLOAD(Op, DAG);
10863 case ISD::STORE: return LowerSTORE(Op, DAG);
10864 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
10865 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10866 case ISD::STRICT_FP_TO_UINT:
10867 case ISD::STRICT_FP_TO_SINT:
10868 case ISD::FP_TO_UINT:
10869 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op));
10870 case ISD::STRICT_UINT_TO_FP:
10871 case ISD::STRICT_SINT_TO_FP:
10872 case ISD::UINT_TO_FP:
10873 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10874 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
10875
10876 // Lower 64-bit shifts.
10877 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG);
10878 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG);
10879 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG);
10880
10881 case ISD::FSHL: return LowerFunnelShift(Op, DAG);
10882 case ISD::FSHR: return LowerFunnelShift(Op, DAG);
10883
10884 // Vector-related lowering.
10885 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
10886 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
10887 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
10888 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
10889 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10890 case ISD::MUL: return LowerMUL(Op, DAG);
10891 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10892 case ISD::STRICT_FP_ROUND:
10893 case ISD::FP_ROUND:
10894 return LowerFP_ROUND(Op, DAG);
10895 case ISD::ROTL: return LowerROTL(Op, DAG);
10896
10897 // For counter-based loop handling.
10898 case ISD::INTRINSIC_W_CHAIN: return SDValue();
10899
10900 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
10901
10902 // Frame & Return address.
10903 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10904 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10905
10906 case ISD::INTRINSIC_VOID:
10907 return LowerINTRINSIC_VOID(Op, DAG);
10908 case ISD::BSWAP:
10909 return LowerBSWAP(Op, DAG);
10910 case ISD::ATOMIC_CMP_SWAP:
10911 return LowerATOMIC_CMP_SWAP(Op, DAG);
10912 }
10913}
10914
10915void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
10916 SmallVectorImpl<SDValue>&Results,
10917 SelectionDAG &DAG) const {
10918 SDLoc dl(N);
10919 switch (N->getOpcode()) {
10920 default:
10921 llvm_unreachable("Do not know how to custom type legalize this operation!")::llvm::llvm_unreachable_internal("Do not know how to custom type legalize this operation!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10921)
;
10922 case ISD::READCYCLECOUNTER: {
10923 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
10924 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0));
10925
10926 Results.push_back(
10927 DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, RTB, RTB.getValue(1)));
10928 Results.push_back(RTB.getValue(2));
10929 break;
10930 }
10931 case ISD::INTRINSIC_W_CHAIN: {
10932 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() !=
10933 Intrinsic::loop_decrement)
10934 break;
10935
10936 assert(N->getValueType(0) == MVT::i1 &&(static_cast <bool> (N->getValueType(0) == MVT::i1 &&
"Unexpected result type for CTR decrement intrinsic") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::i1 && \"Unexpected result type for CTR decrement intrinsic\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10937, __extension__ __PRETTY_FUNCTION__))
10937 "Unexpected result type for CTR decrement intrinsic")(static_cast <bool> (N->getValueType(0) == MVT::i1 &&
"Unexpected result type for CTR decrement intrinsic") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::i1 && \"Unexpected result type for CTR decrement intrinsic\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 10937, __extension__ __PRETTY_FUNCTION__))
;
10938 EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
10939 N->getValueType(0));
10940 SDVTList VTs = DAG.getVTList(SVT, MVT::Other);
10941 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0),
10942 N->getOperand(1));
10943
10944 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewInt));
10945 Results.push_back(NewInt.getValue(1));
10946 break;
10947 }
10948 case ISD::VAARG: {
10949 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64())
10950 return;
10951
10952 EVT VT = N->getValueType(0);
10953
10954 if (VT == MVT::i64) {
10955 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG);
10956
10957 Results.push_back(NewNode);
10958 Results.push_back(NewNode.getValue(1));
10959 }
10960 return;
10961 }
10962 case ISD::STRICT_FP_TO_SINT:
10963 case ISD::STRICT_FP_TO_UINT:
10964 case ISD::FP_TO_SINT:
10965 case ISD::FP_TO_UINT:
10966 // LowerFP_TO_INT() can only handle f32 and f64.
10967 if (N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType() ==
10968 MVT::ppcf128)
10969 return;
10970 Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl));
10971 return;
10972 case ISD::TRUNCATE: {
10973 if (!N->getValueType(0).isVector())
10974 return;
10975 SDValue Lowered = LowerTRUNCATEVector(SDValue(N, 0), DAG);
10976 if (Lowered)
10977 Results.push_back(Lowered);
10978 return;
10979 }
10980 case ISD::FSHL:
10981 case ISD::FSHR:
10982 // Don't handle funnel shifts here.
10983 return;
10984 case ISD::BITCAST:
10985 // Don't handle bitcast here.
10986 return;
10987 case ISD::FP_EXTEND:
10988 SDValue Lowered = LowerFP_EXTEND(SDValue(N, 0), DAG);
10989 if (Lowered)
10990 Results.push_back(Lowered);
10991 return;
10992 }
10993}
10994
10995//===----------------------------------------------------------------------===//
10996// Other Lowering Code
10997//===----------------------------------------------------------------------===//
10998
10999static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
11000 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
11001 Function *Func = Intrinsic::getDeclaration(M, Id);
11002 return Builder.CreateCall(Func, {});
11003}
11004
11005// The mappings for emitLeading/TrailingFence is taken from
11006// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
11007Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
11008 Instruction *Inst,
11009 AtomicOrdering Ord) const {
11010 if (Ord == AtomicOrdering::SequentiallyConsistent)
11011 return callIntrinsic(Builder, Intrinsic::ppc_sync);
11012 if (isReleaseOrStronger(Ord))
11013 return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
11014 return nullptr;
11015}
11016
11017Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
11018 Instruction *Inst,
11019 AtomicOrdering Ord) const {
11020 if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) {
11021 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
11022 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
11023 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
11024 if (isa<LoadInst>(Inst) && Subtarget.isPPC64())
11025 return Builder.CreateCall(
11026 Intrinsic::getDeclaration(
11027 Builder.GetInsertBlock()->getParent()->getParent(),
11028 Intrinsic::ppc_cfence, {Inst->getType()}),
11029 {Inst});
11030 // FIXME: Can use isync for rmw operation.
11031 return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
11032 }
11033 return nullptr;
11034}
11035
11036MachineBasicBlock *
11037PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB,
11038 unsigned AtomicSize,
11039 unsigned BinOpcode,
11040 unsigned CmpOpcode,
11041 unsigned CmpPred) const {
11042 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
11043 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
11044
11045 auto LoadMnemonic = PPC::LDARX;
11046 auto StoreMnemonic = PPC::STDCX;
11047 switch (AtomicSize) {
11048 default:
11049 llvm_unreachable("Unexpected size of atomic entity")::llvm::llvm_unreachable_internal("Unexpected size of atomic entity"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11049)
;
11050 case 1:
11051 LoadMnemonic = PPC::LBARX;
11052 StoreMnemonic = PPC::STBCX;
11053 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4")(static_cast <bool> (Subtarget.hasPartwordAtomics() &&
"Call this only with size >=4") ? void (0) : __assert_fail
("Subtarget.hasPartwordAtomics() && \"Call this only with size >=4\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11053, __extension__ __PRETTY_FUNCTION__))
;
11054 break;
11055 case 2:
11056 LoadMnemonic = PPC::LHARX;
11057 StoreMnemonic = PPC::STHCX;
11058 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4")(static_cast <bool> (Subtarget.hasPartwordAtomics() &&
"Call this only with size >=4") ? void (0) : __assert_fail
("Subtarget.hasPartwordAtomics() && \"Call this only with size >=4\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11058, __extension__ __PRETTY_FUNCTION__))
;
11059 break;
11060 case 4:
11061 LoadMnemonic = PPC::LWARX;
11062 StoreMnemonic = PPC::STWCX;
11063 break;
11064 case 8:
11065 LoadMnemonic = PPC::LDARX;
11066 StoreMnemonic = PPC::STDCX;
11067 break;
11068 }
11069
11070 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11071 MachineFunction *F = BB->getParent();
11072 MachineFunction::iterator It = ++BB->getIterator();
11073
11074 Register dest = MI.getOperand(0).getReg();
11075 Register ptrA = MI.getOperand(1).getReg();
11076 Register ptrB = MI.getOperand(2).getReg();
11077 Register incr = MI.getOperand(3).getReg();
11078 DebugLoc dl = MI.getDebugLoc();
11079
11080 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB);
11081 MachineBasicBlock *loop2MBB =
11082 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr;
11083 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
11084 F->insert(It, loopMBB);
11085 if (CmpOpcode)
11086 F->insert(It, loop2MBB);
11087 F->insert(It, exitMBB);
11088 exitMBB->splice(exitMBB->begin(), BB,
11089 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11090 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
11091
11092 MachineRegisterInfo &RegInfo = F->getRegInfo();
11093 Register TmpReg = (!BinOpcode) ? incr :
11094 RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass
11095 : &PPC::GPRCRegClass);
11096
11097 // thisMBB:
11098 // ...
11099 // fallthrough --> loopMBB
11100 BB->addSuccessor(loopMBB);
11101
11102 // loopMBB:
11103 // l[wd]arx dest, ptr
11104 // add r0, dest, incr
11105 // st[wd]cx. r0, ptr
11106 // bne- loopMBB
11107 // fallthrough --> exitMBB
11108
11109 // For max/min...
11110 // loopMBB:
11111 // l[wd]arx dest, ptr
11112 // cmpl?[wd] incr, dest
11113 // bgt exitMBB
11114 // loop2MBB:
11115 // st[wd]cx. dest, ptr
11116 // bne- loopMBB
11117 // fallthrough --> exitMBB
11118
11119 BB = loopMBB;
11120 BuildMI(BB, dl, TII->get(LoadMnemonic), dest)
11121 .addReg(ptrA).addReg(ptrB);
11122 if (BinOpcode)
11123 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest);
11124 if (CmpOpcode) {
11125 // Signed comparisons of byte or halfword values must be sign-extended.
11126 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) {
11127 Register ExtReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass);
11128 BuildMI(BB, dl, TII->get(AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH),
11129 ExtReg).addReg(dest);
11130 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0)
11131 .addReg(incr).addReg(ExtReg);
11132 } else
11133 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0)
11134 .addReg(incr).addReg(dest);
11135
11136 BuildMI(BB, dl, TII->get(PPC::BCC))
11137 .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB);
11138 BB->addSuccessor(loop2MBB);
11139 BB->addSuccessor(exitMBB);
11140 BB = loop2MBB;
11141 }
11142 BuildMI(BB, dl, TII->get(StoreMnemonic))
11143 .addReg(TmpReg).addReg(ptrA).addReg(ptrB);
11144 BuildMI(BB, dl, TII->get(PPC::BCC))
11145 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB);
11146 BB->addSuccessor(loopMBB);
11147 BB->addSuccessor(exitMBB);
11148
11149 // exitMBB:
11150 // ...
11151 BB = exitMBB;
11152 return BB;
11153}
11154
11155static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) {
11156 switch(MI.getOpcode()) {
11157 default:
11158 return false;
11159 case PPC::COPY:
11160 return TII->isSignExtended(MI);
11161 case PPC::LHA:
11162 case PPC::LHA8:
11163 case PPC::LHAU:
11164 case PPC::LHAU8:
11165 case PPC::LHAUX:
11166 case PPC::LHAUX8:
11167 case PPC::LHAX:
11168 case PPC::LHAX8:
11169 case PPC::LWA:
11170 case PPC::LWAUX:
11171 case PPC::LWAX:
11172 case PPC::LWAX_32:
11173 case PPC::LWA_32:
11174 case PPC::PLHA:
11175 case PPC::PLHA8:
11176 case PPC::PLHA8pc:
11177 case PPC::PLHApc:
11178 case PPC::PLWA:
11179 case PPC::PLWA8:
11180 case PPC::PLWA8pc:
11181 case PPC::PLWApc:
11182 case PPC::EXTSB:
11183 case PPC::EXTSB8:
11184 case PPC::EXTSB8_32_64:
11185 case PPC::EXTSB8_rec:
11186 case PPC::EXTSB_rec:
11187 case PPC::EXTSH:
11188 case PPC::EXTSH8:
11189 case PPC::EXTSH8_32_64:
11190 case PPC::EXTSH8_rec:
11191 case PPC::EXTSH_rec:
11192 case PPC::EXTSW:
11193 case PPC::EXTSWSLI:
11194 case PPC::EXTSWSLI_32_64:
11195 case PPC::EXTSWSLI_32_64_rec:
11196 case PPC::EXTSWSLI_rec:
11197 case PPC::EXTSW_32:
11198 case PPC::EXTSW_32_64:
11199 case PPC::EXTSW_32_64_rec:
11200 case PPC::EXTSW_rec:
11201 case PPC::SRAW:
11202 case PPC::SRAWI:
11203 case PPC::SRAWI_rec:
11204 case PPC::SRAW_rec:
11205 return true;
11206 }
11207 return false;
11208}
11209
11210MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
11211 MachineInstr &MI, MachineBasicBlock *BB,
11212 bool is8bit, // operation
11213 unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const {
11214 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
11215 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
11216
11217 // If this is a signed comparison and the value being compared is not known
11218 // to be sign extended, sign extend it here.
11219 DebugLoc dl = MI.getDebugLoc();
11220 MachineFunction *F = BB->getParent();
11221 MachineRegisterInfo &RegInfo = F->getRegInfo();
11222 Register incr = MI.getOperand(3).getReg();
11223 bool IsSignExtended = Register::isVirtualRegister(incr) &&
11224 isSignExtended(*RegInfo.getVRegDef(incr), TII);
11225
11226 if (CmpOpcode == PPC::CMPW && !IsSignExtended) {
11227 Register ValueReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass);
11228 BuildMI(*BB, MI, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueReg)
11229 .addReg(MI.getOperand(3).getReg());
11230 MI.getOperand(3).setReg(ValueReg);
11231 }
11232 // If we support part-word atomic mnemonics, just use them
11233 if (Subtarget.hasPartwordAtomics())
11234 return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode,
11235 CmpPred);
11236
11237 // In 64 bit mode we have to use 64 bits for addresses, even though the
11238 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address
11239 // registers without caring whether they're 32 or 64, but here we're
11240 // doing actual arithmetic on the addresses.
11241 bool is64bit = Subtarget.isPPC64();
11242 bool isLittleEndian = Subtarget.isLittleEndian();
11243 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
11244
11245 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11246 MachineFunction::iterator It = ++BB->getIterator();
11247
11248 Register dest = MI.getOperand(0).getReg();
11249 Register ptrA = MI.getOperand(1).getReg();
11250 Register ptrB = MI.getOperand(2).getReg();
11251
11252 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB);
11253 MachineBasicBlock *loop2MBB =
11254 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr;
11255 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
11256 F->insert(It, loopMBB);
11257 if (CmpOpcode)
11258 F->insert(It, loop2MBB);
11259 F->insert(It, exitMBB);
11260 exitMBB->splice(exitMBB->begin(), BB,
11261 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11262 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
11263
11264 const TargetRegisterClass *RC =
11265 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
11266 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
11267
11268 Register PtrReg = RegInfo.createVirtualRegister(RC);
11269 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC);
11270 Register ShiftReg =
11271 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC);
11272 Register Incr2Reg = RegInfo.createVirtualRegister(GPRC);
11273 Register MaskReg = RegInfo.createVirtualRegister(GPRC);
11274 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC);
11275 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC);
11276 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC);
11277 Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC);
11278 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC);
11279 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC);
11280 Register SrwDestReg = RegInfo.createVirtualRegister(GPRC);
11281 Register Ptr1Reg;
11282 Register TmpReg =
11283 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC);
11284
11285 // thisMBB:
11286 // ...
11287 // fallthrough --> loopMBB
11288 BB->addSuccessor(loopMBB);
11289
11290 // The 4-byte load must be aligned, while a char or short may be
11291 // anywhere in the word. Hence all this nasty bookkeeping code.
11292 // add ptr1, ptrA, ptrB [copy if ptrA==0]
11293 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
11294 // xori shift, shift1, 24 [16]
11295 // rlwinm ptr, ptr1, 0, 0, 29
11296 // slw incr2, incr, shift
11297 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
11298 // slw mask, mask2, shift
11299 // loopMBB:
11300 // lwarx tmpDest, ptr
11301 // add tmp, tmpDest, incr2
11302 // andc tmp2, tmpDest, mask
11303 // and tmp3, tmp, mask
11304 // or tmp4, tmp3, tmp2
11305 // stwcx. tmp4, ptr
11306 // bne- loopMBB
11307 // fallthrough --> exitMBB
11308 // srw SrwDest, tmpDest, shift
11309 // rlwinm SrwDest, SrwDest, 0, 24 [16], 31
11310 if (ptrA != ZeroReg) {
11311 Ptr1Reg = RegInfo.createVirtualRegister(RC);
11312 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
11313 .addReg(ptrA)
11314 .addReg(ptrB);
11315 } else {
11316 Ptr1Reg = ptrB;
11317 }
11318 // We need use 32-bit subregister to avoid mismatch register class in 64-bit
11319 // mode.
11320 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg)
11321 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0)
11322 .addImm(3)
11323 .addImm(27)
11324 .addImm(is8bit ? 28 : 27);
11325 if (!isLittleEndian)
11326 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg)
11327 .addReg(Shift1Reg)
11328 .addImm(is8bit ? 24 : 16);
11329 if (is64bit)
11330 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg)
11331 .addReg(Ptr1Reg)
11332 .addImm(0)
11333 .addImm(61);
11334 else
11335 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg)
11336 .addReg(Ptr1Reg)
11337 .addImm(0)
11338 .addImm(0)
11339 .addImm(29);
11340 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg);
11341 if (is8bit)
11342 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255);
11343 else {
11344 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0);
11345 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg)
11346 .addReg(Mask3Reg)
11347 .addImm(65535);
11348 }
11349 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg)
11350 .addReg(Mask2Reg)
11351 .addReg(ShiftReg);
11352
11353 BB = loopMBB;
11354 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg)
11355 .addReg(ZeroReg)
11356 .addReg(PtrReg);
11357 if (BinOpcode)
11358 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg)
11359 .addReg(Incr2Reg)
11360 .addReg(TmpDestReg);
11361 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg)
11362 .addReg(TmpDestReg)
11363 .addReg(MaskReg);
11364 BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg);
11365 if (CmpOpcode) {
11366 // For unsigned comparisons, we can directly compare the shifted values.
11367 // For signed comparisons we shift and sign extend.
11368 Register SReg = RegInfo.createVirtualRegister(GPRC);
11369 BuildMI(BB, dl, TII->get(PPC::AND), SReg)
11370 .addReg(TmpDestReg)
11371 .addReg(MaskReg);
11372 unsigned ValueReg = SReg;
11373 unsigned CmpReg = Incr2Reg;
11374 if (CmpOpcode == PPC::CMPW) {
11375 ValueReg = RegInfo.createVirtualRegister(GPRC);
11376 BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg)
11377 .addReg(SReg)
11378 .addReg(ShiftReg);
11379 Register ValueSReg = RegInfo.createVirtualRegister(GPRC);
11380 BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg)
11381 .addReg(ValueReg);
11382 ValueReg = ValueSReg;
11383 CmpReg = incr;
11384 }
11385 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0)
11386 .addReg(CmpReg)
11387 .addReg(ValueReg);
11388 BuildMI(BB, dl, TII->get(PPC::BCC))
11389 .addImm(CmpPred)
11390 .addReg(PPC::CR0)
11391 .addMBB(exitMBB);
11392 BB->addSuccessor(loop2MBB);
11393 BB->addSuccessor(exitMBB);
11394 BB = loop2MBB;
11395 }
11396 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg);
11397 BuildMI(BB, dl, TII->get(PPC::STWCX))
11398 .addReg(Tmp4Reg)
11399 .addReg(ZeroReg)
11400 .addReg(PtrReg);
11401 BuildMI(BB, dl, TII->get(PPC::BCC))
11402 .addImm(PPC::PRED_NE)
11403 .addReg(PPC::CR0)
11404 .addMBB(loopMBB);
11405 BB->addSuccessor(loopMBB);
11406 BB->addSuccessor(exitMBB);
11407
11408 // exitMBB:
11409 // ...
11410 BB = exitMBB;
11411 // Since the shift amount is not a constant, we need to clear
11412 // the upper bits with a separate RLWINM.
11413 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::RLWINM), dest)
11414 .addReg(SrwDestReg)
11415 .addImm(0)
11416 .addImm(is8bit ? 24 : 16)
11417 .addImm(31);
11418 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), SrwDestReg)
11419 .addReg(TmpDestReg)
11420 .addReg(ShiftReg);
11421 return BB;
11422}
11423
11424llvm::MachineBasicBlock *
11425PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
11426 MachineBasicBlock *MBB) const {
11427 DebugLoc DL = MI.getDebugLoc();
11428 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
11429 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo();
11430
11431 MachineFunction *MF = MBB->getParent();
11432 MachineRegisterInfo &MRI = MF->getRegInfo();
11433
11434 const BasicBlock *BB = MBB->getBasicBlock();
11435 MachineFunction::iterator I = ++MBB->getIterator();
11436
11437 Register DstReg = MI.getOperand(0).getReg();
11438 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
11439 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!")(static_cast <bool> (TRI->isTypeLegalForClass(*RC, MVT
::i32) && "Invalid destination!") ? void (0) : __assert_fail
("TRI->isTypeLegalForClass(*RC, MVT::i32) && \"Invalid destination!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11439, __extension__ __PRETTY_FUNCTION__))
;
11440 Register mainDstReg = MRI.createVirtualRegister(RC);
11441 Register restoreDstReg = MRI.createVirtualRegister(RC);
11442
11443 MVT PVT = getPointerTy(MF->getDataLayout());
11444 assert((PVT == MVT::i64 || PVT == MVT::i32) &&(static_cast <bool> ((PVT == MVT::i64 || PVT == MVT::i32
) && "Invalid Pointer Size!") ? void (0) : __assert_fail
("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11445, __extension__ __PRETTY_FUNCTION__))
11445 "Invalid Pointer Size!")(static_cast <bool> ((PVT == MVT::i64 || PVT == MVT::i32
) && "Invalid Pointer Size!") ? void (0) : __assert_fail
("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11445, __extension__ __PRETTY_FUNCTION__))
;
11446 // For v = setjmp(buf), we generate
11447 //
11448 // thisMBB:
11449 // SjLjSetup mainMBB
11450 // bl mainMBB
11451 // v_restore = 1
11452 // b sinkMBB
11453 //
11454 // mainMBB:
11455 // buf[LabelOffset] = LR
11456 // v_main = 0
11457 //
11458 // sinkMBB:
11459 // v = phi(main, restore)
11460 //
11461
11462 MachineBasicBlock *thisMBB = MBB;
11463 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
11464 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
11465 MF->insert(I, mainMBB);
11466 MF->insert(I, sinkMBB);
11467
11468 MachineInstrBuilder MIB;
11469
11470 // Transfer the remainder of BB and its successor edges to sinkMBB.
11471 sinkMBB->splice(sinkMBB->begin(), MBB,
11472 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
11473 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
11474
11475 // Note that the structure of the jmp_buf used here is not compatible
11476 // with that used by libc, and is not designed to be. Specifically, it
11477 // stores only those 'reserved' registers that LLVM does not otherwise
11478 // understand how to spill. Also, by convention, by the time this
11479 // intrinsic is called, Clang has already stored the frame address in the
11480 // first slot of the buffer and stack address in the third. Following the
11481 // X86 target code, we'll store the jump address in the second slot. We also
11482 // need to save the TOC pointer (R2) to handle jumps between shared
11483 // libraries, and that will be stored in the fourth slot. The thread
11484 // identifier (R13) is not affected.
11485
11486 // thisMBB:
11487 const int64_t LabelOffset = 1 * PVT.getStoreSize();
11488 const int64_t TOCOffset = 3 * PVT.getStoreSize();
11489 const int64_t BPOffset = 4 * PVT.getStoreSize();
11490
11491 // Prepare IP either in reg.
11492 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
11493 Register LabelReg = MRI.createVirtualRegister(PtrRC);
11494 Register BufReg = MI.getOperand(1).getReg();
11495
11496 if (Subtarget.is64BitELFABI()) {
11497 setUsesTOCBasePtr(*MBB->getParent());
11498 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD))
11499 .addReg(PPC::X2)
11500 .addImm(TOCOffset)
11501 .addReg(BufReg)
11502 .cloneMemRefs(MI);
11503 }
11504
11505 // Naked functions never have a base pointer, and so we use r1. For all
11506 // other functions, this decision must be delayed until during PEI.
11507 unsigned BaseReg;
11508 if (MF->getFunction().hasFnAttribute(Attribute::Naked))
11509 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
11510 else
11511 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP;
11512
11513 MIB = BuildMI(*thisMBB, MI, DL,
11514 TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW))
11515 .addReg(BaseReg)
11516 .addImm(BPOffset)
11517 .addReg(BufReg)
11518 .cloneMemRefs(MI);
11519
11520 // Setup
11521 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB);
11522 MIB.addRegMask(TRI->getNoPreservedMask());
11523
11524 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1);
11525
11526 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup))
11527 .addMBB(mainMBB);
11528 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB);
11529
11530 thisMBB->addSuccessor(mainMBB, BranchProbability::getZero());
11531 thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne());
11532
11533 // mainMBB:
11534 // mainDstReg = 0
11535 MIB =
11536 BuildMI(mainMBB, DL,
11537 TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg);
11538
11539 // Store IP
11540 if (Subtarget.isPPC64()) {
11541 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD))
11542 .addReg(LabelReg)
11543 .addImm(LabelOffset)
11544 .addReg(BufReg);
11545 } else {
11546 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW))
11547 .addReg(LabelReg)
11548 .addImm(LabelOffset)
11549 .addReg(BufReg);
11550 }
11551 MIB.cloneMemRefs(MI);
11552
11553 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0);
11554 mainMBB->addSuccessor(sinkMBB);
11555
11556 // sinkMBB:
11557 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
11558 TII->get(PPC::PHI), DstReg)
11559 .addReg(mainDstReg).addMBB(mainMBB)
11560 .addReg(restoreDstReg).addMBB(thisMBB);
11561
11562 MI.eraseFromParent();
11563 return sinkMBB;
11564}
11565
11566MachineBasicBlock *
11567PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
11568 MachineBasicBlock *MBB) const {
11569 DebugLoc DL = MI.getDebugLoc();
11570 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
11571
11572 MachineFunction *MF = MBB->getParent();
11573 MachineRegisterInfo &MRI = MF->getRegInfo();
11574
11575 MVT PVT = getPointerTy(MF->getDataLayout());
11576 assert((PVT == MVT::i64 || PVT == MVT::i32) &&(static_cast <bool> ((PVT == MVT::i64 || PVT == MVT::i32
) && "Invalid Pointer Size!") ? void (0) : __assert_fail
("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11577, __extension__ __PRETTY_FUNCTION__))
11577 "Invalid Pointer Size!")(static_cast <bool> ((PVT == MVT::i64 || PVT == MVT::i32
) && "Invalid Pointer Size!") ? void (0) : __assert_fail
("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11577, __extension__ __PRETTY_FUNCTION__))
;
11578
11579 const TargetRegisterClass *RC =
11580 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
11581 Register Tmp = MRI.createVirtualRegister(RC);
11582 // Since FP is only updated here but NOT referenced, it's treated as GPR.
11583 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31;
11584 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1;
11585 unsigned BP =
11586 (PVT == MVT::i64)
11587 ? PPC::X30
11588 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29
11589 : PPC::R30);
11590
11591 MachineInstrBuilder MIB;
11592
11593 const int64_t LabelOffset = 1 * PVT.getStoreSize();
11594 const int64_t SPOffset = 2 * PVT.getStoreSize();
11595 const int64_t TOCOffset = 3 * PVT.getStoreSize();
11596 const int64_t BPOffset = 4 * PVT.getStoreSize();
11597
11598 Register BufReg = MI.getOperand(0).getReg();
11599
11600 // Reload FP (the jumped-to function may not have had a
11601 // frame pointer, and if so, then its r31 will be restored
11602 // as necessary).
11603 if (PVT == MVT::i64) {
11604 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP)
11605 .addImm(0)
11606 .addReg(BufReg);
11607 } else {
11608 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP)
11609 .addImm(0)
11610 .addReg(BufReg);
11611 }
11612 MIB.cloneMemRefs(MI);
11613
11614 // Reload IP
11615 if (PVT == MVT::i64) {
11616 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp)
11617 .addImm(LabelOffset)
11618 .addReg(BufReg);
11619 } else {
11620 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp)
11621 .addImm(LabelOffset)
11622 .addReg(BufReg);
11623 }
11624 MIB.cloneMemRefs(MI);
11625
11626 // Reload SP
11627 if (PVT == MVT::i64) {
11628 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP)
11629 .addImm(SPOffset)
11630 .addReg(BufReg);
11631 } else {
11632 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP)
11633 .addImm(SPOffset)
11634 .addReg(BufReg);
11635 }
11636 MIB.cloneMemRefs(MI);
11637
11638 // Reload BP
11639 if (PVT == MVT::i64) {
11640 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP)
11641 .addImm(BPOffset)
11642 .addReg(BufReg);
11643 } else {
11644 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP)
11645 .addImm(BPOffset)
11646 .addReg(BufReg);
11647 }
11648 MIB.cloneMemRefs(MI);
11649
11650 // Reload TOC
11651 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) {
11652 setUsesTOCBasePtr(*MBB->getParent());
11653 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2)
11654 .addImm(TOCOffset)
11655 .addReg(BufReg)
11656 .cloneMemRefs(MI);
11657 }
11658
11659 // Jump
11660 BuildMI(*MBB, MI, DL,
11661 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp);
11662 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR));
11663
11664 MI.eraseFromParent();
11665 return MBB;
11666}
11667
11668bool PPCTargetLowering::hasInlineStackProbe(MachineFunction &MF) const {
11669 // If the function specifically requests inline stack probes, emit them.
11670 if (MF.getFunction().hasFnAttribute("probe-stack"))
11671 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
11672 "inline-asm";
11673 return false;
11674}
11675
11676unsigned PPCTargetLowering::getStackProbeSize(MachineFunction &MF) const {
11677 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
11678 unsigned StackAlign = TFI->getStackAlignment();
11679 assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) &&(static_cast <bool> (StackAlign >= 1 && isPowerOf2_32
(StackAlign) && "Unexpected stack alignment") ? void (
0) : __assert_fail ("StackAlign >= 1 && isPowerOf2_32(StackAlign) && \"Unexpected stack alignment\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11680, __extension__ __PRETTY_FUNCTION__))
11680 "Unexpected stack alignment")(static_cast <bool> (StackAlign >= 1 && isPowerOf2_32
(StackAlign) && "Unexpected stack alignment") ? void (
0) : __assert_fail ("StackAlign >= 1 && isPowerOf2_32(StackAlign) && \"Unexpected stack alignment\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11680, __extension__ __PRETTY_FUNCTION__))
;
11681 // The default stack probe size is 4096 if the function has no
11682 // stack-probe-size attribute.
11683 unsigned StackProbeSize = 4096;
11684 const Function &Fn = MF.getFunction();
11685 if (Fn.hasFnAttribute("stack-probe-size"))
11686 Fn.getFnAttribute("stack-probe-size")
11687 .getValueAsString()
11688 .getAsInteger(0, StackProbeSize);
11689 // Round down to the stack alignment.
11690 StackProbeSize &= ~(StackAlign - 1);
11691 return StackProbeSize ? StackProbeSize : StackAlign;
11692}
11693
11694// Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted
11695// into three phases. In the first phase, it uses pseudo instruction
11696// PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and
11697// FinalStackPtr. In the second phase, it generates a loop for probing blocks.
11698// At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of
11699// MaxCallFrameSize so that it can calculate correct data area pointer.
11700MachineBasicBlock *
11701PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
11702 MachineBasicBlock *MBB) const {
11703 const bool isPPC64 = Subtarget.isPPC64();
11704 MachineFunction *MF = MBB->getParent();
11705 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
11706 DebugLoc DL = MI.getDebugLoc();
11707 const unsigned ProbeSize = getStackProbeSize(*MF);
11708 const BasicBlock *ProbedBB = MBB->getBasicBlock();
11709 MachineRegisterInfo &MRI = MF->getRegInfo();
11710 // The CFG of probing stack looks as
11711 // +-----+
11712 // | MBB |
11713 // +--+--+
11714 // |
11715 // +----v----+
11716 // +--->+ TestMBB +---+
11717 // | +----+----+ |
11718 // | | |
11719 // | +-----v----+ |
11720 // +---+ BlockMBB | |
11721 // +----------+ |
11722 // |
11723 // +---------+ |
11724 // | TailMBB +<--+
11725 // +---------+
11726 // In MBB, calculate previous frame pointer and final stack pointer.
11727 // In TestMBB, test if sp is equal to final stack pointer, if so, jump to
11728 // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB.
11729 // TailMBB is spliced via \p MI.
11730 MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(ProbedBB);
11731 MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(ProbedBB);
11732 MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(ProbedBB);
11733
11734 MachineFunction::iterator MBBIter = ++MBB->getIterator();
11735 MF->insert(MBBIter, TestMBB);
11736 MF->insert(MBBIter, BlockMBB);
11737 MF->insert(MBBIter, TailMBB);
11738
11739 const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
11740 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
11741
11742 Register DstReg = MI.getOperand(0).getReg();
11743 Register NegSizeReg = MI.getOperand(1).getReg();
11744 Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
11745 Register FinalStackPtr = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11746 Register FramePointer = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11747 Register ActualNegSizeReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11748
11749 // Since value of NegSizeReg might be realigned in prologepilog, insert a
11750 // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and
11751 // NegSize.
11752 unsigned ProbeOpc;
11753 if (!MRI.hasOneNonDBGUse(NegSizeReg))
11754 ProbeOpc =
11755 isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32;
11756 else
11757 // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg
11758 // and NegSizeReg will be allocated in the same phyreg to avoid
11759 // redundant copy when NegSizeReg has only one use which is current MI and
11760 // will be replaced by PREPARE_PROBED_ALLOCA then.
11761 ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64
11762 : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32;
11763 BuildMI(*MBB, {MI}, DL, TII->get(ProbeOpc), FramePointer)
11764 .addDef(ActualNegSizeReg)
11765 .addReg(NegSizeReg)
11766 .add(MI.getOperand(2))
11767 .add(MI.getOperand(3));
11768
11769 // Calculate final stack pointer, which equals to SP + ActualNegSize.
11770 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4),
11771 FinalStackPtr)
11772 .addReg(SPReg)
11773 .addReg(ActualNegSizeReg);
11774
11775 // Materialize a scratch register for update.
11776 int64_t NegProbeSize = -(int64_t)ProbeSize;
11777 assert(isInt<32>(NegProbeSize) && "Unhandled probe size!")(static_cast <bool> (isInt<32>(NegProbeSize) &&
"Unhandled probe size!") ? void (0) : __assert_fail ("isInt<32>(NegProbeSize) && \"Unhandled probe size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 11777, __extension__ __PRETTY_FUNCTION__))
;
11778 Register ScratchReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11779 if (!isInt<16>(NegProbeSize)) {
11780 Register TempReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11781 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::LIS8 : PPC::LIS), TempReg)
11782 .addImm(NegProbeSize >> 16);
11783 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ORI8 : PPC::ORI),
11784 ScratchReg)
11785 .addReg(TempReg)
11786 .addImm(NegProbeSize & 0xFFFF);
11787 } else
11788 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::LI8 : PPC::LI), ScratchReg)
11789 .addImm(NegProbeSize);
11790
11791 {
11792 // Probing leading residual part.
11793 Register Div = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11794 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::DIVD : PPC::DIVW), Div)
11795 .addReg(ActualNegSizeReg)
11796 .addReg(ScratchReg);
11797 Register Mul = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11798 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::MULLD : PPC::MULLW), Mul)
11799 .addReg(Div)
11800 .addReg(ScratchReg);
11801 Register NegMod = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11802 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::SUBF8 : PPC::SUBF), NegMod)
11803 .addReg(Mul)
11804 .addReg(ActualNegSizeReg);
11805 BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
11806 .addReg(FramePointer)
11807 .addReg(SPReg)
11808 .addReg(NegMod);
11809 }
11810
11811 {
11812 // Remaining part should be multiple of ProbeSize.
11813 Register CmpResult = MRI.createVirtualRegister(&PPC::CRRCRegClass);
11814 BuildMI(TestMBB, DL, TII->get(isPPC64 ? PPC::CMPD : PPC::CMPW), CmpResult)
11815 .addReg(SPReg)
11816 .addReg(FinalStackPtr);
11817 BuildMI(TestMBB, DL, TII->get(PPC::BCC))
11818 .addImm(PPC::PRED_EQ)
11819 .addReg(CmpResult)
11820 .addMBB(TailMBB);
11821 TestMBB->addSuccessor(BlockMBB);
11822 TestMBB->addSuccessor(TailMBB);
11823 }
11824
11825 {
11826 // Touch the block.
11827 // |P...|P...|P...
11828 BuildMI(BlockMBB, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
11829 .addReg(FramePointer)
11830 .addReg(SPReg)
11831 .addReg(ScratchReg);
11832 BuildMI(BlockMBB, DL, TII->get(PPC::B)).addMBB(TestMBB);
11833 BlockMBB->addSuccessor(TestMBB);
11834 }
11835
11836 // Calculation of MaxCallFrameSize is deferred to prologepilog, use
11837 // DYNAREAOFFSET pseudo instruction to get the future result.
11838 Register MaxCallFrameSizeReg =
11839 MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
11840 BuildMI(TailMBB, DL,
11841 TII->get(isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET),
11842 MaxCallFrameSizeReg)
11843 .add(MI.getOperand(2))
11844 .add(MI.getOperand(3));
11845 BuildMI(TailMBB, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4), DstReg)
11846 .addReg(SPReg)
11847 .addReg(MaxCallFrameSizeReg);
11848
11849 // Splice instructions after MI to TailMBB.
11850 TailMBB->splice(TailMBB->end(), MBB,
11851 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
11852 TailMBB->transferSuccessorsAndUpdatePHIs(MBB);
11853 MBB->addSuccessor(TestMBB);
11854
11855 // Delete the pseudo instruction.
11856 MI.eraseFromParent();
11857
11858 ++NumDynamicAllocaProbed;
11859 return TailMBB;
11860}
11861
11862MachineBasicBlock *
11863PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
11864 MachineBasicBlock *BB) const {
11865 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
11866 MI.getOpcode() == TargetOpcode::PATCHPOINT) {
11867 if (Subtarget.is64BitELFABI() &&
11868 MI.getOpcode() == TargetOpcode::PATCHPOINT &&
11869 !Subtarget.isUsingPCRelativeCalls()) {
11870 // Call lowering should have added an r2 operand to indicate a dependence
11871 // on the TOC base pointer value. It can't however, because there is no
11872 // way to mark the dependence as implicit there, and so the stackmap code
11873 // will confuse it with a regular operand. Instead, add the dependence
11874 // here.
11875 MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true));
11876 }
11877
11878 return emitPatchPoint(MI, BB);
11879 }
11880
11881 if (MI.getOpcode() == PPC::EH_SjLj_SetJmp32 ||
11882 MI.getOpcode() == PPC::EH_SjLj_SetJmp64) {
11883 return emitEHSjLjSetJmp(MI, BB);
11884 } else if (MI.getOpcode() == PPC::EH_SjLj_LongJmp32 ||
11885 MI.getOpcode() == PPC::EH_SjLj_LongJmp64) {
11886 return emitEHSjLjLongJmp(MI, BB);
11887 }
11888
11889 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
11890
11891 // To "insert" these instructions we actually have to insert their
11892 // control-flow patterns.
11893 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11894 MachineFunction::iterator It = ++BB->getIterator();
11895
11896 MachineFunction *F = BB->getParent();
11897
11898 if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
11899 MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 ||
11900 MI.getOpcode() == PPC::SELECT_I8) {
11901 SmallVector<MachineOperand, 2> Cond;
11902 if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
11903 MI.getOpcode() == PPC::SELECT_CC_I8)
11904 Cond.push_back(MI.getOperand(4));
11905 else
11906 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
11907 Cond.push_back(MI.getOperand(1));
11908
11909 DebugLoc dl = MI.getDebugLoc();
11910 TII->insertSelect(*BB, MI, dl, MI.getOperand(0).getReg(), Cond,
11911 MI.getOperand(2).getReg(), MI.getOperand(3).getReg());
11912 } else if (MI.getOpcode() == PPC::SELECT_CC_F4 ||
11913 MI.getOpcode() == PPC::SELECT_CC_F8 ||
11914 MI.getOpcode() == PPC::SELECT_CC_F16 ||
11915 MI.getOpcode() == PPC::SELECT_CC_VRRC ||
11916 MI.getOpcode() == PPC::SELECT_CC_VSFRC ||
11917 MI.getOpcode() == PPC::SELECT_CC_VSSRC ||
11918 MI.getOpcode() == PPC::SELECT_CC_VSRC ||
11919 MI.getOpcode() == PPC::SELECT_CC_SPE4 ||
11920 MI.getOpcode() == PPC::SELECT_CC_SPE ||
11921 MI.getOpcode() == PPC::SELECT_F4 ||
11922 MI.getOpcode() == PPC::SELECT_F8 ||
11923 MI.getOpcode() == PPC::SELECT_F16 ||
11924 MI.getOpcode() == PPC::SELECT_SPE ||
11925 MI.getOpcode() == PPC::SELECT_SPE4 ||
11926 MI.getOpcode() == PPC::SELECT_VRRC ||
11927 MI.getOpcode() == PPC::SELECT_VSFRC ||
11928 MI.getOpcode() == PPC::SELECT_VSSRC ||
11929 MI.getOpcode() == PPC::SELECT_VSRC) {
11930 // The incoming instruction knows the destination vreg to set, the
11931 // condition code register to branch on, the true/false values to
11932 // select between, and a branch opcode to use.
11933
11934 // thisMBB:
11935 // ...
11936 // TrueVal = ...
11937 // cmpTY ccX, r1, r2
11938 // bCC copy1MBB
11939 // fallthrough --> copy0MBB
11940 MachineBasicBlock *thisMBB = BB;
11941 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
11942 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
11943 DebugLoc dl = MI.getDebugLoc();
11944 F->insert(It, copy0MBB);
11945 F->insert(It, sinkMBB);
11946
11947 // Transfer the remainder of BB and its successor edges to sinkMBB.
11948 sinkMBB->splice(sinkMBB->begin(), BB,
11949 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11950 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
11951
11952 // Next, add the true and fallthrough blocks as its successors.
11953 BB->addSuccessor(copy0MBB);
11954 BB->addSuccessor(sinkMBB);
11955
11956 if (MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 ||
11957 MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 ||
11958 MI.getOpcode() == PPC::SELECT_F16 ||
11959 MI.getOpcode() == PPC::SELECT_SPE4 ||
11960 MI.getOpcode() == PPC::SELECT_SPE ||
11961 MI.getOpcode() == PPC::SELECT_VRRC ||
11962 MI.getOpcode() == PPC::SELECT_VSFRC ||
11963 MI.getOpcode() == PPC::SELECT_VSSRC ||
11964 MI.getOpcode() == PPC::SELECT_VSRC) {
11965 BuildMI(BB, dl, TII->get(PPC::BC))
11966 .addReg(MI.getOperand(1).getReg())
11967 .addMBB(sinkMBB);
11968 } else {
11969 unsigned SelectPred = MI.getOperand(4).getImm();
11970 BuildMI(BB, dl, TII->get(PPC::BCC))
11971 .addImm(SelectPred)
11972 .addReg(MI.getOperand(1).getReg())
11973 .addMBB(sinkMBB);
11974 }
11975
11976 // copy0MBB:
11977 // %FalseValue = ...
11978 // # fallthrough to sinkMBB
11979 BB = copy0MBB;
11980
11981 // Update machine-CFG edges
11982 BB->addSuccessor(sinkMBB);
11983
11984 // sinkMBB:
11985 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
11986 // ...
11987 BB = sinkMBB;
11988 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::PHI), MI.getOperand(0).getReg())
11989 .addReg(MI.getOperand(3).getReg())
11990 .addMBB(copy0MBB)
11991 .addReg(MI.getOperand(2).getReg())
11992 .addMBB(thisMBB);
11993 } else if (MI.getOpcode() == PPC::ReadTB) {
11994 // To read the 64-bit time-base register on a 32-bit target, we read the
11995 // two halves. Should the counter have wrapped while it was being read, we
11996 // need to try again.
11997 // ...
11998 // readLoop:
11999 // mfspr Rx,TBU # load from TBU
12000 // mfspr Ry,TB # load from TB
12001 // mfspr Rz,TBU # load from TBU
12002 // cmpw crX,Rx,Rz # check if 'old'='new'
12003 // bne readLoop # branch if they're not equal
12004 // ...
12005
12006 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB);
12007 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
12008 DebugLoc dl = MI.getDebugLoc();
12009 F->insert(It, readMBB);
12010 F->insert(It, sinkMBB);
12011
12012 // Transfer the remainder of BB and its successor edges to sinkMBB.
12013 sinkMBB->splice(sinkMBB->begin(), BB,
12014 std::next(MachineBasicBlock::iterator(MI)), BB->end());
12015 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
12016
12017 BB->addSuccessor(readMBB);
12018 BB = readMBB;
12019
12020 MachineRegisterInfo &RegInfo = F->getRegInfo();
12021 Register ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass);
12022 Register LoReg = MI.getOperand(0).getReg();
12023 Register HiReg = MI.getOperand(1).getReg();
12024
12025 BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269);
12026 BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268);
12027 BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269);
12028
12029 Register CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass);
12030
12031 BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg)
12032 .addReg(HiReg)
12033 .addReg(ReadAgainReg);
12034 BuildMI(BB, dl, TII->get(PPC::BCC))
12035 .addImm(PPC::PRED_NE)
12036 .addReg(CmpReg)
12037 .addMBB(readMBB);
12038
12039 BB->addSuccessor(readMBB);
12040 BB->addSuccessor(sinkMBB);
12041 } else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I8)
12042 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4);
12043 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I16)
12044 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4);
12045 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I32)
12046 BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4);
12047 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I64)
12048 BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8);
12049
12050 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I8)
12051 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND);
12052 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I16)
12053 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND);
12054 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I32)
12055 BB = EmitAtomicBinary(MI, BB, 4, PPC::AND);
12056 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I64)
12057 BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8);
12058
12059 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I8)
12060 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR);
12061 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I16)
12062 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR);
12063 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I32)
12064 BB = EmitAtomicBinary(MI, BB, 4, PPC::OR);
12065 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I64)
12066 BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8);
12067
12068 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I8)
12069 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR);
12070 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I16)
12071 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR);
12072 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I32)
12073 BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR);
12074 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I64)
12075 BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8);
12076
12077 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I8)
12078 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND);
12079 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I16)
12080 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND);
12081 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I32)
12082 BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND);
12083 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I64)
12084 BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8);
12085
12086 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I8)
12087 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF);
12088 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I16)
12089 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF);
12090 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I32)
12091 BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF);
12092 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I64)
12093 BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8);
12094
12095 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I8)
12096 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_GE);
12097 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I16)
12098 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_GE);
12099 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I32)
12100 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_GE);
12101 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I64)
12102 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_GE);
12103
12104 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I8)
12105 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_LE);
12106 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I16)
12107 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_LE);
12108 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I32)
12109 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_LE);
12110 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I64)
12111 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_LE);
12112
12113 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I8)
12114 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_GE);
12115 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I16)
12116 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_GE);
12117 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I32)
12118 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_GE);
12119 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I64)
12120 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_GE);
12121
12122 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I8)
12123 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_LE);
12124 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I16)
12125 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_LE);
12126 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I32)
12127 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_LE);
12128 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I64)
12129 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_LE);
12130
12131 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I8)
12132 BB = EmitPartwordAtomicBinary(MI, BB, true, 0);
12133 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I16)
12134 BB = EmitPartwordAtomicBinary(MI, BB, false, 0);
12135 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I32)
12136 BB = EmitAtomicBinary(MI, BB, 4, 0);
12137 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I64)
12138 BB = EmitAtomicBinary(MI, BB, 8, 0);
12139 else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
12140 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 ||
12141 (Subtarget.hasPartwordAtomics() &&
12142 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) ||
12143 (Subtarget.hasPartwordAtomics() &&
12144 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) {
12145 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64;
12146
12147 auto LoadMnemonic = PPC::LDARX;
12148 auto StoreMnemonic = PPC::STDCX;
12149 switch (MI.getOpcode()) {
12150 default:
12151 llvm_unreachable("Compare and swap of unknown size")::llvm::llvm_unreachable_internal("Compare and swap of unknown size"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12151)
;
12152 case PPC::ATOMIC_CMP_SWAP_I8:
12153 LoadMnemonic = PPC::LBARX;
12154 StoreMnemonic = PPC::STBCX;
12155 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.")(static_cast <bool> (Subtarget.hasPartwordAtomics() &&
"No support partword atomics.") ? void (0) : __assert_fail (
"Subtarget.hasPartwordAtomics() && \"No support partword atomics.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12155, __extension__ __PRETTY_FUNCTION__))
;
12156 break;
12157 case PPC::ATOMIC_CMP_SWAP_I16:
12158 LoadMnemonic = PPC::LHARX;
12159 StoreMnemonic = PPC::STHCX;
12160 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.")(static_cast <bool> (Subtarget.hasPartwordAtomics() &&
"No support partword atomics.") ? void (0) : __assert_fail (
"Subtarget.hasPartwordAtomics() && \"No support partword atomics.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12160, __extension__ __PRETTY_FUNCTION__))
;
12161 break;
12162 case PPC::ATOMIC_CMP_SWAP_I32:
12163 LoadMnemonic = PPC::LWARX;
12164 StoreMnemonic = PPC::STWCX;
12165 break;
12166 case PPC::ATOMIC_CMP_SWAP_I64:
12167 LoadMnemonic = PPC::LDARX;
12168 StoreMnemonic = PPC::STDCX;
12169 break;
12170 }
12171 Register dest = MI.getOperand(0).getReg();
12172 Register ptrA = MI.getOperand(1).getReg();
12173 Register ptrB = MI.getOperand(2).getReg();
12174 Register oldval = MI.getOperand(3).getReg();
12175 Register newval = MI.getOperand(4).getReg();
12176 DebugLoc dl = MI.getDebugLoc();
12177
12178 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB);
12179 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB);
12180 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB);
12181 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
12182 F->insert(It, loop1MBB);
12183 F->insert(It, loop2MBB);
12184 F->insert(It, midMBB);
12185 F->insert(It, exitMBB);
12186 exitMBB->splice(exitMBB->begin(), BB,
12187 std::next(MachineBasicBlock::iterator(MI)), BB->end());
12188 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
12189
12190 // thisMBB:
12191 // ...
12192 // fallthrough --> loopMBB
12193 BB->addSuccessor(loop1MBB);
12194
12195 // loop1MBB:
12196 // l[bhwd]arx dest, ptr
12197 // cmp[wd] dest, oldval
12198 // bne- midMBB
12199 // loop2MBB:
12200 // st[bhwd]cx. newval, ptr
12201 // bne- loopMBB
12202 // b exitBB
12203 // midMBB:
12204 // st[bhwd]cx. dest, ptr
12205 // exitBB:
12206 BB = loop1MBB;
12207 BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB);
12208 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0)
12209 .addReg(oldval)
12210 .addReg(dest);
12211 BuildMI(BB, dl, TII->get(PPC::BCC))
12212 .addImm(PPC::PRED_NE)
12213 .addReg(PPC::CR0)
12214 .addMBB(midMBB);
12215 BB->addSuccessor(loop2MBB);
12216 BB->addSuccessor(midMBB);
12217
12218 BB = loop2MBB;
12219 BuildMI(BB, dl, TII->get(StoreMnemonic))
12220 .addReg(newval)
12221 .addReg(ptrA)
12222 .addReg(ptrB);
12223 BuildMI(BB, dl, TII->get(PPC::BCC))
12224 .addImm(PPC::PRED_NE)
12225 .addReg(PPC::CR0)
12226 .addMBB(loop1MBB);
12227 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB);
12228 BB->addSuccessor(loop1MBB);
12229 BB->addSuccessor(exitMBB);
12230
12231 BB = midMBB;
12232 BuildMI(BB, dl, TII->get(StoreMnemonic))
12233 .addReg(dest)
12234 .addReg(ptrA)
12235 .addReg(ptrB);
12236 BB->addSuccessor(exitMBB);
12237
12238 // exitMBB:
12239 // ...
12240 BB = exitMBB;
12241 } else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 ||
12242 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) {
12243 // We must use 64-bit registers for addresses when targeting 64-bit,
12244 // since we're actually doing arithmetic on them. Other registers
12245 // can be 32-bit.
12246 bool is64bit = Subtarget.isPPC64();
12247 bool isLittleEndian = Subtarget.isLittleEndian();
12248 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8;
12249
12250 Register dest = MI.getOperand(0).getReg();
12251 Register ptrA = MI.getOperand(1).getReg();
12252 Register ptrB = MI.getOperand(2).getReg();
12253 Register oldval = MI.getOperand(3).getReg();
12254 Register newval = MI.getOperand(4).getReg();
12255 DebugLoc dl = MI.getDebugLoc();
12256
12257 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB);
12258 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB);
12259 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB);
12260 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
12261 F->insert(It, loop1MBB);
12262 F->insert(It, loop2MBB);
12263 F->insert(It, midMBB);
12264 F->insert(It, exitMBB);
12265 exitMBB->splice(exitMBB->begin(), BB,
12266 std::next(MachineBasicBlock::iterator(MI)), BB->end());
12267 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
12268
12269 MachineRegisterInfo &RegInfo = F->getRegInfo();
12270 const TargetRegisterClass *RC =
12271 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
12272 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
12273
12274 Register PtrReg = RegInfo.createVirtualRegister(RC);
12275 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC);
12276 Register ShiftReg =
12277 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC);
12278 Register NewVal2Reg = RegInfo.createVirtualRegister(GPRC);
12279 Register NewVal3Reg = RegInfo.createVirtualRegister(GPRC);
12280 Register OldVal2Reg = RegInfo.createVirtualRegister(GPRC);
12281 Register OldVal3Reg = RegInfo.createVirtualRegister(GPRC);
12282 Register MaskReg = RegInfo.createVirtualRegister(GPRC);
12283 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC);
12284 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC);
12285 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC);
12286 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC);
12287 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC);
12288 Register Ptr1Reg;
12289 Register TmpReg = RegInfo.createVirtualRegister(GPRC);
12290 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
12291 // thisMBB:
12292 // ...
12293 // fallthrough --> loopMBB
12294 BB->addSuccessor(loop1MBB);
12295
12296 // The 4-byte load must be aligned, while a char or short may be
12297 // anywhere in the word. Hence all this nasty bookkeeping code.
12298 // add ptr1, ptrA, ptrB [copy if ptrA==0]
12299 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
12300 // xori shift, shift1, 24 [16]
12301 // rlwinm ptr, ptr1, 0, 0, 29
12302 // slw newval2, newval, shift
12303 // slw oldval2, oldval,shift
12304 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
12305 // slw mask, mask2, shift
12306 // and newval3, newval2, mask
12307 // and oldval3, oldval2, mask
12308 // loop1MBB:
12309 // lwarx tmpDest, ptr
12310 // and tmp, tmpDest, mask
12311 // cmpw tmp, oldval3
12312 // bne- midMBB
12313 // loop2MBB:
12314 // andc tmp2, tmpDest, mask
12315 // or tmp4, tmp2, newval3
12316 // stwcx. tmp4, ptr
12317 // bne- loop1MBB
12318 // b exitBB
12319 // midMBB:
12320 // stwcx. tmpDest, ptr
12321 // exitBB:
12322 // srw dest, tmpDest, shift
12323 if (ptrA != ZeroReg) {
12324 Ptr1Reg = RegInfo.createVirtualRegister(RC);
12325 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
12326 .addReg(ptrA)
12327 .addReg(ptrB);
12328 } else {
12329 Ptr1Reg = ptrB;
12330 }
12331
12332 // We need use 32-bit subregister to avoid mismatch register class in 64-bit
12333 // mode.
12334 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg)
12335 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0)
12336 .addImm(3)
12337 .addImm(27)
12338 .addImm(is8bit ? 28 : 27);
12339 if (!isLittleEndian)
12340 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg)
12341 .addReg(Shift1Reg)
12342 .addImm(is8bit ? 24 : 16);
12343 if (is64bit)
12344 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg)
12345 .addReg(Ptr1Reg)
12346 .addImm(0)
12347 .addImm(61);
12348 else
12349 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg)
12350 .addReg(Ptr1Reg)
12351 .addImm(0)
12352 .addImm(0)
12353 .addImm(29);
12354 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg)
12355 .addReg(newval)
12356 .addReg(ShiftReg);
12357 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg)
12358 .addReg(oldval)
12359 .addReg(ShiftReg);
12360 if (is8bit)
12361 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255);
12362 else {
12363 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0);
12364 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg)
12365 .addReg(Mask3Reg)
12366 .addImm(65535);
12367 }
12368 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg)
12369 .addReg(Mask2Reg)
12370 .addReg(ShiftReg);
12371 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg)
12372 .addReg(NewVal2Reg)
12373 .addReg(MaskReg);
12374 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg)
12375 .addReg(OldVal2Reg)
12376 .addReg(MaskReg);
12377
12378 BB = loop1MBB;
12379 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg)
12380 .addReg(ZeroReg)
12381 .addReg(PtrReg);
12382 BuildMI(BB, dl, TII->get(PPC::AND), TmpReg)
12383 .addReg(TmpDestReg)
12384 .addReg(MaskReg);
12385 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0)
12386 .addReg(TmpReg)
12387 .addReg(OldVal3Reg);
12388 BuildMI(BB, dl, TII->get(PPC::BCC))
12389 .addImm(PPC::PRED_NE)
12390 .addReg(PPC::CR0)
12391 .addMBB(midMBB);
12392 BB->addSuccessor(loop2MBB);
12393 BB->addSuccessor(midMBB);
12394
12395 BB = loop2MBB;
12396 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg)
12397 .addReg(TmpDestReg)
12398 .addReg(MaskReg);
12399 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg)
12400 .addReg(Tmp2Reg)
12401 .addReg(NewVal3Reg);
12402 BuildMI(BB, dl, TII->get(PPC::STWCX))
12403 .addReg(Tmp4Reg)
12404 .addReg(ZeroReg)
12405 .addReg(PtrReg);
12406 BuildMI(BB, dl, TII->get(PPC::BCC))
12407 .addImm(PPC::PRED_NE)
12408 .addReg(PPC::CR0)
12409 .addMBB(loop1MBB);
12410 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB);
12411 BB->addSuccessor(loop1MBB);
12412 BB->addSuccessor(exitMBB);
12413
12414 BB = midMBB;
12415 BuildMI(BB, dl, TII->get(PPC::STWCX))
12416 .addReg(TmpDestReg)
12417 .addReg(ZeroReg)
12418 .addReg(PtrReg);
12419 BB->addSuccessor(exitMBB);
12420
12421 // exitMBB:
12422 // ...
12423 BB = exitMBB;
12424 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest)
12425 .addReg(TmpReg)
12426 .addReg(ShiftReg);
12427 } else if (MI.getOpcode() == PPC::FADDrtz) {
12428 // This pseudo performs an FADD with rounding mode temporarily forced
12429 // to round-to-zero. We emit this via custom inserter since the FPSCR
12430 // is not modeled at the SelectionDAG level.
12431 Register Dest = MI.getOperand(0).getReg();
12432 Register Src1 = MI.getOperand(1).getReg();
12433 Register Src2 = MI.getOperand(2).getReg();
12434 DebugLoc dl = MI.getDebugLoc();
12435
12436 MachineRegisterInfo &RegInfo = F->getRegInfo();
12437 Register MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass);
12438
12439 // Save FPSCR value.
12440 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg);
12441
12442 // Set rounding mode to round-to-zero.
12443 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1))
12444 .addImm(31)
12445 .addReg(PPC::RM, RegState::ImplicitDefine);
12446
12447 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0))
12448 .addImm(30)
12449 .addReg(PPC::RM, RegState::ImplicitDefine);
12450
12451 // Perform addition.
12452 auto MIB = BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest)
12453 .addReg(Src1)
12454 .addReg(Src2);
12455 if (MI.getFlag(MachineInstr::NoFPExcept))
12456 MIB.setMIFlag(MachineInstr::NoFPExcept);
12457
12458 // Restore FPSCR value.
12459 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg);
12460 } else if (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT ||
12461 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT ||
12462 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 ||
12463 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) {
12464 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 ||
12465 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8)
12466 ? PPC::ANDI8_rec
12467 : PPC::ANDI_rec;
12468 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT ||
12469 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8);
12470
12471 MachineRegisterInfo &RegInfo = F->getRegInfo();
12472 Register Dest = RegInfo.createVirtualRegister(
12473 Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass);
12474
12475 DebugLoc Dl = MI.getDebugLoc();
12476 BuildMI(*BB, MI, Dl, TII->get(Opcode), Dest)
12477 .addReg(MI.getOperand(1).getReg())
12478 .addImm(1);
12479 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY),
12480 MI.getOperand(0).getReg())
12481 .addReg(IsEQ ? PPC::CR0EQ : PPC::CR0GT);
12482 } else if (MI.getOpcode() == PPC::TCHECK_RET) {
12483 DebugLoc Dl = MI.getDebugLoc();
12484 MachineRegisterInfo &RegInfo = F->getRegInfo();
12485 Register CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass);
12486 BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg);
12487 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY),
12488 MI.getOperand(0).getReg())
12489 .addReg(CRReg);
12490 } else if (MI.getOpcode() == PPC::TBEGIN_RET) {
12491 DebugLoc Dl = MI.getDebugLoc();
12492 unsigned Imm = MI.getOperand(1).getImm();
12493 BuildMI(*BB, MI, Dl, TII->get(PPC::TBEGIN)).addImm(Imm);
12494 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY),
12495 MI.getOperand(0).getReg())
12496 .addReg(PPC::CR0EQ);
12497 } else if (MI.getOpcode() == PPC::SETRNDi) {
12498 DebugLoc dl = MI.getDebugLoc();
12499 Register OldFPSCRReg = MI.getOperand(0).getReg();
12500
12501 // Save FPSCR value.
12502 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg);
12503
12504 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has
12505 // the following settings:
12506 // 00 Round to nearest
12507 // 01 Round to 0
12508 // 10 Round to +inf
12509 // 11 Round to -inf
12510
12511 // When the operand is immediate, using the two least significant bits of
12512 // the immediate to set the bits 62:63 of FPSCR.
12513 unsigned Mode = MI.getOperand(1).getImm();
12514 BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0))
12515 .addImm(31)
12516 .addReg(PPC::RM, RegState::ImplicitDefine);
12517
12518 BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0))
12519 .addImm(30)
12520 .addReg(PPC::RM, RegState::ImplicitDefine);
12521 } else if (MI.getOpcode() == PPC::SETRND) {
12522 DebugLoc dl = MI.getDebugLoc();
12523
12524 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg
12525 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg.
12526 // If the target doesn't have DirectMove, we should use stack to do the
12527 // conversion, because the target doesn't have the instructions like mtvsrd
12528 // or mfvsrd to do this conversion directly.
12529 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) {
12530 if (Subtarget.hasDirectMove()) {
12531 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), DestReg)
12532 .addReg(SrcReg);
12533 } else {
12534 // Use stack to do the register copy.
12535 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD;
12536 MachineRegisterInfo &RegInfo = F->getRegInfo();
12537 const TargetRegisterClass *RC = RegInfo.getRegClass(SrcReg);
12538 if (RC == &PPC::F8RCRegClass) {
12539 // Copy register from F8RCRegClass to G8RCRegclass.
12540 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) &&(static_cast <bool> ((RegInfo.getRegClass(DestReg) == &
PPC::G8RCRegClass) && "Unsupported RegClass.") ? void
(0) : __assert_fail ("(RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && \"Unsupported RegClass.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12541, __extension__ __PRETTY_FUNCTION__))
12541 "Unsupported RegClass.")(static_cast <bool> ((RegInfo.getRegClass(DestReg) == &
PPC::G8RCRegClass) && "Unsupported RegClass.") ? void
(0) : __assert_fail ("(RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && \"Unsupported RegClass.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12541, __extension__ __PRETTY_FUNCTION__))
;
12542
12543 StoreOp = PPC::STFD;
12544 LoadOp = PPC::LD;
12545 } else {
12546 // Copy register from G8RCRegClass to F8RCRegclass.
12547 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) &&(static_cast <bool> ((RegInfo.getRegClass(SrcReg) == &
PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) ==
&PPC::F8RCRegClass) && "Unsupported RegClass.") ?
void (0) : __assert_fail ("(RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && \"Unsupported RegClass.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12549, __extension__ __PRETTY_FUNCTION__))
12548 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) &&(static_cast <bool> ((RegInfo.getRegClass(SrcReg) == &
PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) ==
&PPC::F8RCRegClass) && "Unsupported RegClass.") ?
void (0) : __assert_fail ("(RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && \"Unsupported RegClass.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12549, __extension__ __PRETTY_FUNCTION__))
12549 "Unsupported RegClass.")(static_cast <bool> ((RegInfo.getRegClass(SrcReg) == &
PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) ==
&PPC::F8RCRegClass) && "Unsupported RegClass.") ?
void (0) : __assert_fail ("(RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && \"Unsupported RegClass.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12549, __extension__ __PRETTY_FUNCTION__))
;
12550 }
12551
12552 MachineFrameInfo &MFI = F->getFrameInfo();
12553 int FrameIdx = MFI.CreateStackObject(8, Align(8), false);
12554
12555 MachineMemOperand *MMOStore = F->getMachineMemOperand(
12556 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
12557 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
12558 MFI.getObjectAlign(FrameIdx));
12559
12560 // Store the SrcReg into the stack.
12561 BuildMI(*BB, MI, dl, TII->get(StoreOp))
12562 .addReg(SrcReg)
12563 .addImm(0)
12564 .addFrameIndex(FrameIdx)
12565 .addMemOperand(MMOStore);
12566
12567 MachineMemOperand *MMOLoad = F->getMachineMemOperand(
12568 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
12569 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
12570 MFI.getObjectAlign(FrameIdx));
12571
12572 // Load from the stack where SrcReg is stored, and save to DestReg,
12573 // so we have done the RegClass conversion from RegClass::SrcReg to
12574 // RegClass::DestReg.
12575 BuildMI(*BB, MI, dl, TII->get(LoadOp), DestReg)
12576 .addImm(0)
12577 .addFrameIndex(FrameIdx)
12578 .addMemOperand(MMOLoad);
12579 }
12580 };
12581
12582 Register OldFPSCRReg = MI.getOperand(0).getReg();
12583
12584 // Save FPSCR value.
12585 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg);
12586
12587 // When the operand is gprc register, use two least significant bits of the
12588 // register and mtfsf instruction to set the bits 62:63 of FPSCR.
12589 //
12590 // copy OldFPSCRTmpReg, OldFPSCRReg
12591 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1)
12592 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62
12593 // copy NewFPSCRReg, NewFPSCRTmpReg
12594 // mtfsf 255, NewFPSCRReg
12595 MachineOperand SrcOp = MI.getOperand(1);
12596 MachineRegisterInfo &RegInfo = F->getRegInfo();
12597 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass);
12598
12599 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg);
12600
12601 Register ImDefReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass);
12602 Register ExtSrcReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass);
12603
12604 // The first operand of INSERT_SUBREG should be a register which has
12605 // subregisters, we only care about its RegClass, so we should use an
12606 // IMPLICIT_DEF register.
12607 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), ImDefReg);
12608 BuildMI(*BB, MI, dl, TII->get(PPC::INSERT_SUBREG), ExtSrcReg)
12609 .addReg(ImDefReg)
12610 .add(SrcOp)
12611 .addImm(1);
12612
12613 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass);
12614 BuildMI(*BB, MI, dl, TII->get(PPC::RLDIMI), NewFPSCRTmpReg)
12615 .addReg(OldFPSCRTmpReg)
12616 .addReg(ExtSrcReg)
12617 .addImm(0)
12618 .addImm(62);
12619
12620 Register NewFPSCRReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass);
12621 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg);
12622
12623 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63
12624 // bits of FPSCR.
12625 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF))
12626 .addImm(255)
12627 .addReg(NewFPSCRReg)
12628 .addImm(0)
12629 .addImm(0);
12630 } else if (MI.getOpcode() == PPC::SETFLM) {
12631 DebugLoc Dl = MI.getDebugLoc();
12632
12633 // Result of setflm is previous FPSCR content, so we need to save it first.
12634 Register OldFPSCRReg = MI.getOperand(0).getReg();
12635 BuildMI(*BB, MI, Dl, TII->get(PPC::MFFS), OldFPSCRReg);
12636
12637 // Put bits in 32:63 to FPSCR.
12638 Register NewFPSCRReg = MI.getOperand(1).getReg();
12639 BuildMI(*BB, MI, Dl, TII->get(PPC::MTFSF))
12640 .addImm(255)
12641 .addReg(NewFPSCRReg)
12642 .addImm(0)
12643 .addImm(0);
12644 } else if (MI.getOpcode() == PPC::PROBED_ALLOCA_32 ||
12645 MI.getOpcode() == PPC::PROBED_ALLOCA_64) {
12646 return emitProbedAlloca(MI, BB);
12647 } else if (MI.getOpcode() == PPC::SPLIT_QUADWORD) {
12648 DebugLoc DL = MI.getDebugLoc();
12649 Register Src = MI.getOperand(2).getReg();
12650 Register Lo = MI.getOperand(0).getReg();
12651 Register Hi = MI.getOperand(1).getReg();
12652 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY))
12653 .addDef(Lo)
12654 .addUse(Src, 0, PPC::sub_gp8_x1);
12655 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY))
12656 .addDef(Hi)
12657 .addUse(Src, 0, PPC::sub_gp8_x0);
12658 } else {
12659 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12659)
;
12660 }
12661
12662 MI.eraseFromParent(); // The pseudo instruction is gone now.
12663 return BB;
12664}
12665
12666//===----------------------------------------------------------------------===//
12667// Target Optimization Hooks
12668//===----------------------------------------------------------------------===//
12669
12670static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
12671 // For the estimates, convergence is quadratic, so we essentially double the
12672 // number of digits correct after every iteration. For both FRE and FRSQRTE,
12673 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
12674 // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
12675 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
12676 if (VT.getScalarType() == MVT::f64)
12677 RefinementSteps++;
12678 return RefinementSteps;
12679}
12680
12681SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
12682 const DenormalMode &Mode) const {
12683 // We only have VSX Vector Test for software Square Root.
12684 EVT VT = Op.getValueType();
12685 if (!isTypeLegal(MVT::i1) ||
12686 (VT != MVT::f64 &&
12687 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())))
12688 return TargetLowering::getSqrtInputTest(Op, DAG, Mode);
12689
12690 SDLoc DL(Op);
12691 // The output register of FTSQRT is CR field.
12692 SDValue FTSQRT = DAG.getNode(PPCISD::FTSQRT, DL, MVT::i32, Op);
12693 // ftsqrt BF,FRB
12694 // Let e_b be the unbiased exponent of the double-precision
12695 // floating-point operand in register FRB.
12696 // fe_flag is set to 1 if either of the following conditions occurs.
12697 // - The double-precision floating-point operand in register FRB is a zero,
12698 // a NaN, or an infinity, or a negative value.
12699 // - e_b is less than or equal to -970.
12700 // Otherwise fe_flag is set to 0.
12701 // Both VSX and non-VSX versions would set EQ bit in the CR if the number is
12702 // not eligible for iteration. (zero/negative/infinity/nan or unbiased
12703 // exponent is less than -970)
12704 SDValue SRIdxVal = DAG.getTargetConstant(PPC::sub_eq, DL, MVT::i32);
12705 return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::i1,
12706 FTSQRT, SRIdxVal),
12707 0);
12708}
12709
12710SDValue
12711PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op,
12712 SelectionDAG &DAG) const {
12713 // We only have VSX Vector Square Root.
12714 EVT VT = Op.getValueType();
12715 if (VT != MVT::f64 &&
12716 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))
12717 return TargetLowering::getSqrtResultForDenormInput(Op, DAG);
12718
12719 return DAG.getNode(PPCISD::FSQRT, SDLoc(Op), VT, Op);
12720}
12721
12722SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
12723 int Enabled, int &RefinementSteps,
12724 bool &UseOneConstNR,
12725 bool Reciprocal) const {
12726 EVT VT = Operand.getValueType();
12727 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
12728 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
12729 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
12730 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
12731 if (RefinementSteps == ReciprocalEstimate::Unspecified)
12732 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
12733
12734 // The Newton-Raphson computation with a single constant does not provide
12735 // enough accuracy on some CPUs.
12736 UseOneConstNR = !Subtarget.needsTwoConstNR();
12737 return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
12738 }
12739 return SDValue();
12740}
12741
12742SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
12743 int Enabled,
12744 int &RefinementSteps) const {
12745 EVT VT = Operand.getValueType();
12746 if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
12747 (VT == MVT::f64 && Subtarget.hasFRE()) ||
12748 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
12749 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
12750 if (RefinementSteps == ReciprocalEstimate::Unspecified)
12751 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
12752 return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
12753 }
12754 return SDValue();
12755}
12756
12757unsigned PPCTargetLowering::combineRepeatedFPDivisors() const {
12758 // Note: This functionality is used only when unsafe-fp-math is enabled, and
12759 // on cores with reciprocal estimates (which are used when unsafe-fp-math is
12760 // enabled for division), this functionality is redundant with the default
12761 // combiner logic (once the division -> reciprocal/multiply transformation
12762 // has taken place). As a result, this matters more for older cores than for
12763 // newer ones.
12764
12765 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
12766 // reciprocal if there are two or more FDIVs (for embedded cores with only
12767 // one FP pipeline) for three or more FDIVs (for generic OOO cores).
12768 switch (Subtarget.getCPUDirective()) {
12769 default:
12770 return 3;
12771 case PPC::DIR_440:
12772 case PPC::DIR_A2:
12773 case PPC::DIR_E500:
12774 case PPC::DIR_E500mc:
12775 case PPC::DIR_E5500:
12776 return 2;
12777 }
12778}
12779
12780// isConsecutiveLSLoc needs to work even if all adds have not yet been
12781// collapsed, and so we need to look through chains of them.
12782static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
12783 int64_t& Offset, SelectionDAG &DAG) {
12784 if (DAG.isBaseWithConstantOffset(Loc)) {
12785 Base = Loc.getOperand(0);
12786 Offset += cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue();
12787
12788 // The base might itself be a base plus an offset, and if so, accumulate
12789 // that as well.
12790 getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG);
12791 }
12792}
12793
12794static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
12795 unsigned Bytes, int Dist,
12796 SelectionDAG &DAG) {
12797 if (VT.getSizeInBits() / 8 != Bytes)
12798 return false;
12799
12800 SDValue BaseLoc = Base->getBasePtr();
12801 if (Loc.getOpcode() == ISD::FrameIndex) {
12802 if (BaseLoc.getOpcode() != ISD::FrameIndex)
12803 return false;
12804 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
12805 int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
12806 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
12807 int FS = MFI.getObjectSize(FI);
12808 int BFS = MFI.getObjectSize(BFI);
12809 if (FS != BFS || FS != (int)Bytes) return false;
12810 return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes);
12811 }
12812
12813 SDValue Base1 = Loc, Base2 = BaseLoc;
12814 int64_t Offset1 = 0, Offset2 = 0;
12815 getBaseWithConstantOffset(Loc, Base1, Offset1, DAG);
12816 getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG);
12817 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes))
12818 return true;
12819
12820 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12821 const GlobalValue *GV1 = nullptr;
12822 const GlobalValue *GV2 = nullptr;
12823 Offset1 = 0;
12824 Offset2 = 0;
12825 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1);
12826 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
12827 if (isGA1 && isGA2 && GV1 == GV2)
12828 return Offset1 == (Offset2 + Dist*Bytes);
12829 return false;
12830}
12831
12832// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does
12833// not enforce equality of the chain operands.
12834static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
12835 unsigned Bytes, int Dist,
12836 SelectionDAG &DAG) {
12837 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N)) {
12838 EVT VT = LS->getMemoryVT();
12839 SDValue Loc = LS->getBasePtr();
12840 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG);
12841 }
12842
12843 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
12844 EVT VT;
12845 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
12846 default: return false;
12847 case Intrinsic::ppc_altivec_lvx:
12848 case Intrinsic::ppc_altivec_lvxl:
12849 case Intrinsic::ppc_vsx_lxvw4x:
12850 case Intrinsic::ppc_vsx_lxvw4x_be:
12851 VT = MVT::v4i32;
12852 break;
12853 case Intrinsic::ppc_vsx_lxvd2x:
12854 case Intrinsic::ppc_vsx_lxvd2x_be:
12855 VT = MVT::v2f64;
12856 break;
12857 case Intrinsic::ppc_altivec_lvebx:
12858 VT = MVT::i8;
12859 break;
12860 case Intrinsic::ppc_altivec_lvehx:
12861 VT = MVT::i16;
12862 break;
12863 case Intrinsic::ppc_altivec_lvewx:
12864 VT = MVT::i32;
12865 break;
12866 }
12867
12868 return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG);
12869 }
12870
12871 if (N->getOpcode() == ISD::INTRINSIC_VOID) {
12872 EVT VT;
12873 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
12874 default: return false;
12875 case Intrinsic::ppc_altivec_stvx:
12876 case Intrinsic::ppc_altivec_stvxl:
12877 case Intrinsic::ppc_vsx_stxvw4x:
12878 VT = MVT::v4i32;
12879 break;
12880 case Intrinsic::ppc_vsx_stxvd2x:
12881 VT = MVT::v2f64;
12882 break;
12883 case Intrinsic::ppc_vsx_stxvw4x_be:
12884 VT = MVT::v4i32;
12885 break;
12886 case Intrinsic::ppc_vsx_stxvd2x_be:
12887 VT = MVT::v2f64;
12888 break;
12889 case Intrinsic::ppc_altivec_stvebx:
12890 VT = MVT::i8;
12891 break;
12892 case Intrinsic::ppc_altivec_stvehx:
12893 VT = MVT::i16;
12894 break;
12895 case Intrinsic::ppc_altivec_stvewx:
12896 VT = MVT::i32;
12897 break;
12898 }
12899
12900 return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG);
12901 }
12902
12903 return false;
12904}
12905
12906// Return true is there is a nearyby consecutive load to the one provided
12907// (regardless of alignment). We search up and down the chain, looking though
12908// token factors and other loads (but nothing else). As a result, a true result
12909// indicates that it is safe to create a new consecutive load adjacent to the
12910// load provided.
12911static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
12912 SDValue Chain = LD->getChain();
12913 EVT VT = LD->getMemoryVT();
12914
12915 SmallSet<SDNode *, 16> LoadRoots;
12916 SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
12917 SmallSet<SDNode *, 16> Visited;
12918
12919 // First, search up the chain, branching to follow all token-factor operands.
12920 // If we find a consecutive load, then we're done, otherwise, record all
12921 // nodes just above the top-level loads and token factors.
12922 while (!Queue.empty()) {
12923 SDNode *ChainNext = Queue.pop_back_val();
12924 if (!Visited.insert(ChainNext).second)
12925 continue;
12926
12927 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(ChainNext)) {
12928 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG))
12929 return true;
12930
12931 if (!Visited.count(ChainLD->getChain().getNode()))
12932 Queue.push_back(ChainLD->getChain().getNode());
12933 } else if (ChainNext->getOpcode() == ISD::TokenFactor) {
12934 for (const SDUse &O : ChainNext->ops())
12935 if (!Visited.count(O.getNode()))
12936 Queue.push_back(O.getNode());
12937 } else
12938 LoadRoots.insert(ChainNext);
12939 }
12940
12941 // Second, search down the chain, starting from the top-level nodes recorded
12942 // in the first phase. These top-level nodes are the nodes just above all
12943 // loads and token factors. Starting with their uses, recursively look though
12944 // all loads (just the chain uses) and token factors to find a consecutive
12945 // load.
12946 Visited.clear();
12947 Queue.clear();
12948
12949 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(),
12950 IE = LoadRoots.end(); I != IE; ++I) {
12951 Queue.push_back(*I);
12952
12953 while (!Queue.empty()) {
12954 SDNode *LoadRoot = Queue.pop_back_val();
12955 if (!Visited.insert(LoadRoot).second)
12956 continue;
12957
12958 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(LoadRoot))
12959 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG))
12960 return true;
12961
12962 for (SDNode::use_iterator UI = LoadRoot->use_begin(),
12963 UE = LoadRoot->use_end(); UI != UE; ++UI)
12964 if (((isa<MemSDNode>(*UI) &&
12965 cast<MemSDNode>(*UI)->getChain().getNode() == LoadRoot) ||
12966 UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI))
12967 Queue.push_back(*UI);
12968 }
12969 }
12970
12971 return false;
12972}
12973
12974/// This function is called when we have proved that a SETCC node can be replaced
12975/// by subtraction (and other supporting instructions) so that the result of
12976/// comparison is kept in a GPR instead of CR. This function is purely for
12977/// codegen purposes and has some flags to guide the codegen process.
12978static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement,
12979 bool Swap, SDLoc &DL, SelectionDAG &DAG) {
12980 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.")(static_cast <bool> (N->getOpcode() == ISD::SETCC &&
"ISD::SETCC Expected.") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SETCC && \"ISD::SETCC Expected.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 12980, __extension__ __PRETTY_FUNCTION__))
;
12981
12982 // Zero extend the operands to the largest legal integer. Originally, they
12983 // must be of a strictly smaller size.
12984 auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0),
12985 DAG.getConstant(Size, DL, MVT::i32));
12986 auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1),
12987 DAG.getConstant(Size, DL, MVT::i32));
12988
12989 // Swap if needed. Depends on the condition code.
12990 if (Swap)
12991 std::swap(Op0, Op1);
12992
12993 // Subtract extended integers.
12994 auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1);
12995
12996 // Move the sign bit to the least significant position and zero out the rest.
12997 // Now the least significant bit carries the result of original comparison.
12998 auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode,
12999 DAG.getConstant(Size - 1, DL, MVT::i32));
13000 auto Final = Shifted;
13001
13002 // Complement the result if needed. Based on the condition code.
13003 if (Complement)
13004 Final = DAG.getNode(ISD::XOR, DL, MVT::i64, Shifted,
13005 DAG.getConstant(1, DL, MVT::i64));
13006
13007 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Final);
13008}
13009
13010SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N,
13011 DAGCombinerInfo &DCI) const {
13012 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.")(static_cast <bool> (N->getOpcode() == ISD::SETCC &&
"ISD::SETCC Expected.") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SETCC && \"ISD::SETCC Expected.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13012, __extension__ __PRETTY_FUNCTION__))
;
13013
13014 SelectionDAG &DAG = DCI.DAG;
13015 SDLoc DL(N);
13016
13017 // Size of integers being compared has a critical role in the following
13018 // analysis, so we prefer to do this when all types are legal.
13019 if (!DCI.isAfterLegalizeDAG())
13020 return SDValue();
13021
13022 // If all users of SETCC extend its value to a legal integer type
13023 // then we replace SETCC with a subtraction
13024 for (SDNode::use_iterator UI = N->use_begin(),
13025 UE = N->use_end(); UI != UE; ++UI) {
13026 if (UI->getOpcode() != ISD::ZERO_EXTEND)
13027 return SDValue();
13028 }
13029
13030 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
13031 auto OpSize = N->getOperand(0).getValueSizeInBits();
13032
13033 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits();
13034
13035 if (OpSize < Size) {
13036 switch (CC) {
13037 default: break;
13038 case ISD::SETULT:
13039 return generateEquivalentSub(N, Size, false, false, DL, DAG);
13040 case ISD::SETULE:
13041 return generateEquivalentSub(N, Size, true, true, DL, DAG);
13042 case ISD::SETUGT:
13043 return generateEquivalentSub(N, Size, false, true, DL, DAG);
13044 case ISD::SETUGE:
13045 return generateEquivalentSub(N, Size, true, false, DL, DAG);
13046 }
13047 }
13048
13049 return SDValue();
13050}
13051
13052SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
13053 DAGCombinerInfo &DCI) const {
13054 SelectionDAG &DAG = DCI.DAG;
13055 SDLoc dl(N);
13056
13057 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits")(static_cast <bool> (Subtarget.useCRBits() && "Expecting to be tracking CR bits"
) ? void (0) : __assert_fail ("Subtarget.useCRBits() && \"Expecting to be tracking CR bits\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13057, __extension__ __PRETTY_FUNCTION__))
;
13058 // If we're tracking CR bits, we need to be careful that we don't have:
13059 // trunc(binary-ops(zext(x), zext(y)))
13060 // or
13061 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...)
13062 // such that we're unnecessarily moving things into GPRs when it would be
13063 // better to keep them in CR bits.
13064
13065 // Note that trunc here can be an actual i1 trunc, or can be the effective
13066 // truncation that comes from a setcc or select_cc.
13067 if (N->getOpcode() == ISD::TRUNCATE &&
13068 N->getValueType(0) != MVT::i1)
13069 return SDValue();
13070
13071 if (N->getOperand(0).getValueType() != MVT::i32 &&
13072 N->getOperand(0).getValueType() != MVT::i64)
13073 return SDValue();
13074
13075 if (N->getOpcode() == ISD::SETCC ||
13076 N->getOpcode() == ISD::SELECT_CC) {
13077 // If we're looking at a comparison, then we need to make sure that the
13078 // high bits (all except for the first) don't matter the result.
13079 ISD::CondCode CC =
13080 cast<CondCodeSDNode>(N->getOperand(
13081 N->getOpcode() == ISD::SETCC ? 2 : 4))->get();
13082 unsigned OpBits = N->getOperand(0).getValueSizeInBits();
13083
13084 if (ISD::isSignedIntSetCC(CC)) {
13085 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits ||
13086 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits)
13087 return SDValue();
13088 } else if (ISD::isUnsignedIntSetCC(CC)) {
13089 if (!DAG.MaskedValueIsZero(N->getOperand(0),
13090 APInt::getHighBitsSet(OpBits, OpBits-1)) ||
13091 !DAG.MaskedValueIsZero(N->getOperand(1),
13092 APInt::getHighBitsSet(OpBits, OpBits-1)))
13093 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI)
13094 : SDValue());
13095 } else {
13096 // This is neither a signed nor an unsigned comparison, just make sure
13097 // that the high bits are equal.
13098 KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0));
13099 KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1));
13100
13101 // We don't really care about what is known about the first bit (if
13102 // anything), so pretend that it is known zero for both to ensure they can
13103 // be compared as constants.
13104 Op1Known.Zero.setBit(0); Op1Known.One.clearBit(0);
13105 Op2Known.Zero.setBit(0); Op2Known.One.clearBit(0);
13106
13107 if (!Op1Known.isConstant() || !Op2Known.isConstant() ||
13108 Op1Known.getConstant() != Op2Known.getConstant())
13109 return SDValue();
13110 }
13111 }
13112
13113 // We now know that the higher-order bits are irrelevant, we just need to
13114 // make sure that all of the intermediate operations are bit operations, and
13115 // all inputs are extensions.
13116 if (N->getOperand(0).getOpcode() != ISD::AND &&
13117 N->getOperand(0).getOpcode() != ISD::OR &&
13118 N->getOperand(0).getOpcode() != ISD::XOR &&
13119 N->getOperand(0).getOpcode() != ISD::SELECT &&
13120 N->getOperand(0).getOpcode() != ISD::SELECT_CC &&
13121 N->getOperand(0).getOpcode() != ISD::TRUNCATE &&
13122 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND &&
13123 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
13124 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND)
13125 return SDValue();
13126
13127 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) &&
13128 N->getOperand(1).getOpcode() != ISD::AND &&
13129 N->getOperand(1).getOpcode() != ISD::OR &&
13130 N->getOperand(1).getOpcode() != ISD::XOR &&
13131 N->getOperand(1).getOpcode() != ISD::SELECT &&
13132 N->getOperand(1).getOpcode() != ISD::SELECT_CC &&
13133 N->getOperand(1).getOpcode() != ISD::TRUNCATE &&
13134 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND &&
13135 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND &&
13136 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND)
13137 return SDValue();
13138
13139 SmallVector<SDValue, 4> Inputs;
13140 SmallVector<SDValue, 8> BinOps, PromOps;
13141 SmallPtrSet<SDNode *, 16> Visited;
13142
13143 for (unsigned i = 0; i < 2; ++i) {
13144 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
13145 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
13146 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
13147 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) ||
13148 isa<ConstantSDNode>(N->getOperand(i)))
13149 Inputs.push_back(N->getOperand(i));
13150 else
13151 BinOps.push_back(N->getOperand(i));
13152
13153 if (N->getOpcode() == ISD::TRUNCATE)
13154 break;
13155 }
13156
13157 // Visit all inputs, collect all binary operations (and, or, xor and
13158 // select) that are all fed by extensions.
13159 while (!BinOps.empty()) {
13160 SDValue BinOp = BinOps.pop_back_val();
13161
13162 if (!Visited.insert(BinOp.getNode()).second)
13163 continue;
13164
13165 PromOps.push_back(BinOp);
13166
13167 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
13168 // The condition of the select is not promoted.
13169 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
13170 continue;
13171 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
13172 continue;
13173
13174 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
13175 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
13176 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
13177 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) ||
13178 isa<ConstantSDNode>(BinOp.getOperand(i))) {
13179 Inputs.push_back(BinOp.getOperand(i));
13180 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
13181 BinOp.getOperand(i).getOpcode() == ISD::OR ||
13182 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
13183 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
13184 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC ||
13185 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
13186 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
13187 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
13188 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) {
13189 BinOps.push_back(BinOp.getOperand(i));
13190 } else {
13191 // We have an input that is not an extension or another binary
13192 // operation; we'll abort this transformation.
13193 return SDValue();
13194 }
13195 }
13196 }
13197
13198 // Make sure that this is a self-contained cluster of operations (which
13199 // is not quite the same thing as saying that everything has only one
13200 // use).
13201 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
13202 if (isa<ConstantSDNode>(Inputs[i]))
13203 continue;
13204
13205 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(),
13206 UE = Inputs[i].getNode()->use_end();
13207 UI != UE; ++UI) {
13208 SDNode *User = *UI;
13209 if (User != N && !Visited.count(User))
13210 return SDValue();
13211
13212 // Make sure that we're not going to promote the non-output-value
13213 // operand(s) or SELECT or SELECT_CC.
13214 // FIXME: Although we could sometimes handle this, and it does occur in
13215 // practice that one of the condition inputs to the select is also one of
13216 // the outputs, we currently can't deal with this.
13217 if (User->getOpcode() == ISD::SELECT) {
13218 if (User->getOperand(0) == Inputs[i])
13219 return SDValue();
13220 } else if (User->getOpcode() == ISD::SELECT_CC) {
13221 if (User->getOperand(0) == Inputs[i] ||
13222 User->getOperand(1) == Inputs[i])
13223 return SDValue();
13224 }
13225 }
13226 }
13227
13228 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
13229 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(),
13230 UE = PromOps[i].getNode()->use_end();
13231 UI != UE; ++UI) {
13232 SDNode *User = *UI;
13233 if (User != N && !Visited.count(User))
13234 return SDValue();
13235
13236 // Make sure that we're not going to promote the non-output-value
13237 // operand(s) or SELECT or SELECT_CC.
13238 // FIXME: Although we could sometimes handle this, and it does occur in
13239 // practice that one of the condition inputs to the select is also one of
13240 // the outputs, we currently can't deal with this.
13241 if (User->getOpcode() == ISD::SELECT) {
13242 if (User->getOperand(0) == PromOps[i])
13243 return SDValue();
13244 } else if (User->getOpcode() == ISD::SELECT_CC) {
13245 if (User->getOperand(0) == PromOps[i] ||
13246 User->getOperand(1) == PromOps[i])
13247 return SDValue();
13248 }
13249 }
13250 }
13251
13252 // Replace all inputs with the extension operand.
13253 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
13254 // Constants may have users outside the cluster of to-be-promoted nodes,
13255 // and so we need to replace those as we do the promotions.
13256 if (isa<ConstantSDNode>(Inputs[i]))
13257 continue;
13258 else
13259 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0));
13260 }
13261
13262 std::list<HandleSDNode> PromOpHandles;
13263 for (auto &PromOp : PromOps)
13264 PromOpHandles.emplace_back(PromOp);
13265
13266 // Replace all operations (these are all the same, but have a different
13267 // (i1) return type). DAG.getNode will validate that the types of
13268 // a binary operator match, so go through the list in reverse so that
13269 // we've likely promoted both operands first. Any intermediate truncations or
13270 // extensions disappear.
13271 while (!PromOpHandles.empty()) {
13272 SDValue PromOp = PromOpHandles.back().getValue();
13273 PromOpHandles.pop_back();
13274
13275 if (PromOp.getOpcode() == ISD::TRUNCATE ||
13276 PromOp.getOpcode() == ISD::SIGN_EXTEND ||
13277 PromOp.getOpcode() == ISD::ZERO_EXTEND ||
13278 PromOp.getOpcode() == ISD::ANY_EXTEND) {
13279 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) &&
13280 PromOp.getOperand(0).getValueType() != MVT::i1) {
13281 // The operand is not yet ready (see comment below).
13282 PromOpHandles.emplace_front(PromOp);
13283 continue;
13284 }
13285
13286 SDValue RepValue = PromOp.getOperand(0);
13287 if (isa<ConstantSDNode>(RepValue))
13288 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue);
13289
13290 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue);
13291 continue;
13292 }
13293
13294 unsigned C;
13295 switch (PromOp.getOpcode()) {
13296 default: C = 0; break;
13297 case ISD::SELECT: C = 1; break;
13298 case ISD::SELECT_CC: C = 2; break;
13299 }
13300
13301 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) &&
13302 PromOp.getOperand(C).getValueType() != MVT::i1) ||
13303 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) &&
13304 PromOp.getOperand(C+1).getValueType() != MVT::i1)) {
13305 // The to-be-promoted operands of this node have not yet been
13306 // promoted (this should be rare because we're going through the
13307 // list backward, but if one of the operands has several users in
13308 // this cluster of to-be-promoted nodes, it is possible).
13309 PromOpHandles.emplace_front(PromOp);
13310 continue;
13311 }
13312
13313 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(),
13314 PromOp.getNode()->op_end());
13315
13316 // If there are any constant inputs, make sure they're replaced now.
13317 for (unsigned i = 0; i < 2; ++i)
13318 if (isa<ConstantSDNode>(Ops[C+i]))
13319 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]);
13320
13321 DAG.ReplaceAllUsesOfValueWith(PromOp,
13322 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops));
13323 }
13324
13325 // Now we're left with the initial truncation itself.
13326 if (N->getOpcode() == ISD::TRUNCATE)
13327 return N->getOperand(0);
13328
13329 // Otherwise, this is a comparison. The operands to be compared have just
13330 // changed type (to i1), but everything else is the same.
13331 return SDValue(N, 0);
13332}
13333
13334SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
13335 DAGCombinerInfo &DCI) const {
13336 SelectionDAG &DAG = DCI.DAG;
13337 SDLoc dl(N);
13338
13339 // If we're tracking CR bits, we need to be careful that we don't have:
13340 // zext(binary-ops(trunc(x), trunc(y)))
13341 // or
13342 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...)
13343 // such that we're unnecessarily moving things into CR bits that can more
13344 // efficiently stay in GPRs. Note that if we're not certain that the high
13345 // bits are set as required by the final extension, we still may need to do
13346 // some masking to get the proper behavior.
13347
13348 // This same functionality is important on PPC64 when dealing with
13349 // 32-to-64-bit extensions; these occur often when 32-bit values are used as
13350 // the return values of functions. Because it is so similar, it is handled
13351 // here as well.
13352
13353 if (N->getValueType(0) != MVT::i32 &&
13354 N->getValueType(0) != MVT::i64)
13355 return SDValue();
13356
13357 if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) ||
13358 (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64())))
13359 return SDValue();
13360
13361 if (N->getOperand(0).getOpcode() != ISD::AND &&
13362 N->getOperand(0).getOpcode() != ISD::OR &&
13363 N->getOperand(0).getOpcode() != ISD::XOR &&
13364 N->getOperand(0).getOpcode() != ISD::SELECT &&
13365 N->getOperand(0).getOpcode() != ISD::SELECT_CC)
13366 return SDValue();
13367
13368 SmallVector<SDValue, 4> Inputs;
13369 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps;
13370 SmallPtrSet<SDNode *, 16> Visited;
13371
13372 // Visit all inputs, collect all binary operations (and, or, xor and
13373 // select) that are all fed by truncations.
13374 while (!BinOps.empty()) {
13375 SDValue BinOp = BinOps.pop_back_val();
13376
13377 if (!Visited.insert(BinOp.getNode()).second)
13378 continue;
13379
13380 PromOps.push_back(BinOp);
13381
13382 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
13383 // The condition of the select is not promoted.
13384 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
13385 continue;
13386 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
13387 continue;
13388
13389 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
13390 isa<ConstantSDNode>(BinOp.getOperand(i))) {
13391 Inputs.push_back(BinOp.getOperand(i));
13392 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
13393 BinOp.getOperand(i).getOpcode() == ISD::OR ||
13394 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
13395 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
13396 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) {
13397 BinOps.push_back(BinOp.getOperand(i));
13398 } else {
13399 // We have an input that is not a truncation or another binary
13400 // operation; we'll abort this transformation.
13401 return SDValue();
13402 }
13403 }
13404 }
13405
13406 // The operands of a select that must be truncated when the select is
13407 // promoted because the operand is actually part of the to-be-promoted set.
13408 DenseMap<SDNode *, EVT> SelectTruncOp[2];
13409
13410 // Make sure that this is a self-contained cluster of operations (which
13411 // is not quite the same thing as saying that everything has only one
13412 // use).
13413 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
13414 if (isa<ConstantSDNode>(Inputs[i]))
13415 continue;
13416
13417 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(),
13418 UE = Inputs[i].getNode()->use_end();
13419 UI != UE; ++UI) {
13420 SDNode *User = *UI;
13421 if (User != N && !Visited.count(User))
13422 return SDValue();
13423
13424 // If we're going to promote the non-output-value operand(s) or SELECT or
13425 // SELECT_CC, record them for truncation.
13426 if (User->getOpcode() == ISD::SELECT) {
13427 if (User->getOperand(0) == Inputs[i])
13428 SelectTruncOp[0].insert(std::make_pair(User,
13429 User->getOperand(0).getValueType()));
13430 } else if (User->getOpcode() == ISD::SELECT_CC) {
13431 if (User->getOperand(0) == Inputs[i])
13432 SelectTruncOp[0].insert(std::make_pair(User,
13433 User->getOperand(0).getValueType()));
13434 if (User->getOperand(1) == Inputs[i])
13435 SelectTruncOp[1].insert(std::make_pair(User,
13436 User->getOperand(1).getValueType()));
13437 }
13438 }
13439 }
13440
13441 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
13442 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(),
13443 UE = PromOps[i].getNode()->use_end();
13444 UI != UE; ++UI) {
13445 SDNode *User = *UI;
13446 if (User != N && !Visited.count(User))
13447 return SDValue();
13448
13449 // If we're going to promote the non-output-value operand(s) or SELECT or
13450 // SELECT_CC, record them for truncation.
13451 if (User->getOpcode() == ISD::SELECT) {
13452 if (User->getOperand(0) == PromOps[i])
13453 SelectTruncOp[0].insert(std::make_pair(User,
13454 User->getOperand(0).getValueType()));
13455 } else if (User->getOpcode() == ISD::SELECT_CC) {
13456 if (User->getOperand(0) == PromOps[i])
13457 SelectTruncOp[0].insert(std::make_pair(User,
13458 User->getOperand(0).getValueType()));
13459 if (User->getOperand(1) == PromOps[i])
13460 SelectTruncOp[1].insert(std::make_pair(User,
13461 User->getOperand(1).getValueType()));
13462 }
13463 }
13464 }
13465
13466 unsigned PromBits = N->getOperand(0).getValueSizeInBits();
13467 bool ReallyNeedsExt = false;
13468 if (N->getOpcode() != ISD::ANY_EXTEND) {
13469 // If all of the inputs are not already sign/zero extended, then
13470 // we'll still need to do that at the end.
13471 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
13472 if (isa<ConstantSDNode>(Inputs[i]))
13473 continue;
13474
13475 unsigned OpBits =
13476 Inputs[i].getOperand(0).getValueSizeInBits();
13477 assert(PromBits < OpBits && "Truncation not to a smaller bit count?")(static_cast <bool> (PromBits < OpBits && "Truncation not to a smaller bit count?"
) ? void (0) : __assert_fail ("PromBits < OpBits && \"Truncation not to a smaller bit count?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13477, __extension__ __PRETTY_FUNCTION__))
;
13478
13479 if ((N->getOpcode() == ISD::ZERO_EXTEND &&
13480 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0),
13481 APInt::getHighBitsSet(OpBits,
13482 OpBits-PromBits))) ||
13483 (N->getOpcode() == ISD::SIGN_EXTEND &&
13484 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) <
13485 (OpBits-(PromBits-1)))) {
13486 ReallyNeedsExt = true;
13487 break;
13488 }
13489 }
13490 }
13491
13492 // Replace all inputs, either with the truncation operand, or a
13493 // truncation or extension to the final output type.
13494 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
13495 // Constant inputs need to be replaced with the to-be-promoted nodes that
13496 // use them because they might have users outside of the cluster of
13497 // promoted nodes.
13498 if (isa<ConstantSDNode>(Inputs[i]))
13499 continue;
13500
13501 SDValue InSrc = Inputs[i].getOperand(0);
13502 if (Inputs[i].getValueType() == N->getValueType(0))
13503 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc);
13504 else if (N->getOpcode() == ISD::SIGN_EXTEND)
13505 DAG.ReplaceAllUsesOfValueWith(Inputs[i],
13506 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0)));
13507 else if (N->getOpcode() == ISD::ZERO_EXTEND)
13508 DAG.ReplaceAllUsesOfValueWith(Inputs[i],
13509 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0)));
13510 else
13511 DAG.ReplaceAllUsesOfValueWith(Inputs[i],
13512 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0)));
13513 }
13514
13515 std::list<HandleSDNode> PromOpHandles;
13516 for (auto &PromOp : PromOps)
13517 PromOpHandles.emplace_back(PromOp);
13518
13519 // Replace all operations (these are all the same, but have a different
13520 // (promoted) return type). DAG.getNode will validate that the types of
13521 // a binary operator match, so go through the list in reverse so that
13522 // we've likely promoted both operands first.
13523 while (!PromOpHandles.empty()) {
13524 SDValue PromOp = PromOpHandles.back().getValue();
13525 PromOpHandles.pop_back();
13526
13527 unsigned C;
13528 switch (PromOp.getOpcode()) {
13529 default: C = 0; break;
13530 case ISD::SELECT: C = 1; break;
13531 case ISD::SELECT_CC: C = 2; break;
13532 }
13533
13534 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) &&
13535 PromOp.getOperand(C).getValueType() != N->getValueType(0)) ||
13536 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) &&
13537 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) {
13538 // The to-be-promoted operands of this node have not yet been
13539 // promoted (this should be rare because we're going through the
13540 // list backward, but if one of the operands has several users in
13541 // this cluster of to-be-promoted nodes, it is possible).
13542 PromOpHandles.emplace_front(PromOp);
13543 continue;
13544 }
13545
13546 // For SELECT and SELECT_CC nodes, we do a similar check for any
13547 // to-be-promoted comparison inputs.
13548 if (PromOp.getOpcode() == ISD::SELECT ||
13549 PromOp.getOpcode() == ISD::SELECT_CC) {
13550 if ((SelectTruncOp[0].count(PromOp.getNode()) &&
13551 PromOp.getOperand(0).getValueType() != N->getValueType(0)) ||
13552 (SelectTruncOp[1].count(PromOp.getNode()) &&
13553 PromOp.getOperand(1).getValueType() != N->getValueType(0))) {
13554 PromOpHandles.emplace_front(PromOp);
13555 continue;
13556 }
13557 }
13558
13559 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(),
13560 PromOp.getNode()->op_end());
13561
13562 // If this node has constant inputs, then they'll need to be promoted here.
13563 for (unsigned i = 0; i < 2; ++i) {
13564 if (!isa<ConstantSDNode>(Ops[C+i]))
13565 continue;
13566 if (Ops[C+i].getValueType() == N->getValueType(0))
13567 continue;
13568
13569 if (N->getOpcode() == ISD::SIGN_EXTEND)
13570 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0));
13571 else if (N->getOpcode() == ISD::ZERO_EXTEND)
13572 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0));
13573 else
13574 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0));
13575 }
13576
13577 // If we've promoted the comparison inputs of a SELECT or SELECT_CC,
13578 // truncate them again to the original value type.
13579 if (PromOp.getOpcode() == ISD::SELECT ||
13580 PromOp.getOpcode() == ISD::SELECT_CC) {
13581 auto SI0 = SelectTruncOp[0].find(PromOp.getNode());
13582 if (SI0 != SelectTruncOp[0].end())
13583 Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]);
13584 auto SI1 = SelectTruncOp[1].find(PromOp.getNode());
13585 if (SI1 != SelectTruncOp[1].end())
13586 Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]);
13587 }
13588
13589 DAG.ReplaceAllUsesOfValueWith(PromOp,
13590 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops));
13591 }
13592
13593 // Now we're left with the initial extension itself.
13594 if (!ReallyNeedsExt)
13595 return N->getOperand(0);
13596
13597 // To zero extend, just mask off everything except for the first bit (in the
13598 // i1 case).
13599 if (N->getOpcode() == ISD::ZERO_EXTEND)
13600 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0),
13601 DAG.getConstant(APInt::getLowBitsSet(
13602 N->getValueSizeInBits(0), PromBits),
13603 dl, N->getValueType(0)));
13604
13605 assert(N->getOpcode() == ISD::SIGN_EXTEND &&(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
&& "Invalid extension type") ? void (0) : __assert_fail
("N->getOpcode() == ISD::SIGN_EXTEND && \"Invalid extension type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13606, __extension__ __PRETTY_FUNCTION__))
13606 "Invalid extension type")(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
&& "Invalid extension type") ? void (0) : __assert_fail
("N->getOpcode() == ISD::SIGN_EXTEND && \"Invalid extension type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13606, __extension__ __PRETTY_FUNCTION__))
;
13607 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout());
13608 SDValue ShiftCst =
13609 DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy);
13610 return DAG.getNode(
13611 ISD::SRA, dl, N->getValueType(0),
13612 DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst),
13613 ShiftCst);
13614}
13615
13616SDValue PPCTargetLowering::combineSetCC(SDNode *N,
13617 DAGCombinerInfo &DCI) const {
13618 assert(N->getOpcode() == ISD::SETCC &&(static_cast <bool> (N->getOpcode() == ISD::SETCC &&
"Should be called with a SETCC node") ? void (0) : __assert_fail
("N->getOpcode() == ISD::SETCC && \"Should be called with a SETCC node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13619, __extension__ __PRETTY_FUNCTION__))
13619 "Should be called with a SETCC node")(static_cast <bool> (N->getOpcode() == ISD::SETCC &&
"Should be called with a SETCC node") ? void (0) : __assert_fail
("N->getOpcode() == ISD::SETCC && \"Should be called with a SETCC node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13619, __extension__ __PRETTY_FUNCTION__))
;
13620
13621 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
13622 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
13623 SDValue LHS = N->getOperand(0);
13624 SDValue RHS = N->getOperand(1);
13625
13626 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS.
13627 if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
13628 LHS.hasOneUse())
13629 std::swap(LHS, RHS);
13630
13631 // x == 0-y --> x+y == 0
13632 // x != 0-y --> x+y != 0
13633 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) &&
13634 RHS.hasOneUse()) {
13635 SDLoc DL(N);
13636 SelectionDAG &DAG = DCI.DAG;
13637 EVT VT = N->getValueType(0);
13638 EVT OpVT = LHS.getValueType();
13639 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1));
13640 return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC);
13641 }
13642 }
13643
13644 return DAGCombineTruncBoolExt(N, DCI);
13645}
13646
13647// Is this an extending load from an f32 to an f64?
13648static bool isFPExtLoad(SDValue Op) {
13649 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode()))
13650 return LD->getExtensionType() == ISD::EXTLOAD &&
13651 Op.getValueType() == MVT::f64;
13652 return false;
13653}
13654
13655/// Reduces the number of fp-to-int conversion when building a vector.
13656///
13657/// If this vector is built out of floating to integer conversions,
13658/// transform it to a vector built out of floating point values followed by a
13659/// single floating to integer conversion of the vector.
13660/// Namely (build_vector (fptosi $A), (fptosi $B), ...)
13661/// becomes (fptosi (build_vector ($A, $B, ...)))
13662SDValue PPCTargetLowering::
13663combineElementTruncationToVectorTruncation(SDNode *N,
13664 DAGCombinerInfo &DCI) const {
13665 assert(N->getOpcode() == ISD::BUILD_VECTOR &&(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "Should be called with a BUILD_VECTOR node") ? void
(0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"Should be called with a BUILD_VECTOR node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13666, __extension__ __PRETTY_FUNCTION__))
13666 "Should be called with a BUILD_VECTOR node")(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "Should be called with a BUILD_VECTOR node") ? void
(0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"Should be called with a BUILD_VECTOR node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13666, __extension__ __PRETTY_FUNCTION__))
;
13667
13668 SelectionDAG &DAG = DCI.DAG;
13669 SDLoc dl(N);
13670
13671 SDValue FirstInput = N->getOperand(0);
13672 assert(FirstInput.getOpcode() == PPCISD::MFVSR &&(static_cast <bool> (FirstInput.getOpcode() == PPCISD::
MFVSR && "The input operand must be an fp-to-int conversion."
) ? void (0) : __assert_fail ("FirstInput.getOpcode() == PPCISD::MFVSR && \"The input operand must be an fp-to-int conversion.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13673, __extension__ __PRETTY_FUNCTION__))
13673 "The input operand must be an fp-to-int conversion.")(static_cast <bool> (FirstInput.getOpcode() == PPCISD::
MFVSR && "The input operand must be an fp-to-int conversion."
) ? void (0) : __assert_fail ("FirstInput.getOpcode() == PPCISD::MFVSR && \"The input operand must be an fp-to-int conversion.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13673, __extension__ __PRETTY_FUNCTION__))
;
13674
13675 // This combine happens after legalization so the fp_to_[su]i nodes are
13676 // already converted to PPCSISD nodes.
13677 unsigned FirstConversion = FirstInput.getOperand(0).getOpcode();
13678 if (FirstConversion == PPCISD::FCTIDZ ||
13679 FirstConversion == PPCISD::FCTIDUZ ||
13680 FirstConversion == PPCISD::FCTIWZ ||
13681 FirstConversion == PPCISD::FCTIWUZ) {
13682 bool IsSplat = true;
13683 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ ||
13684 FirstConversion == PPCISD::FCTIWUZ;
13685 EVT SrcVT = FirstInput.getOperand(0).getValueType();
13686 SmallVector<SDValue, 4> Ops;
13687 EVT TargetVT = N->getValueType(0);
13688 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
13689 SDValue NextOp = N->getOperand(i);
13690 if (NextOp.getOpcode() != PPCISD::MFVSR)
13691 return SDValue();
13692 unsigned NextConversion = NextOp.getOperand(0).getOpcode();
13693 if (NextConversion != FirstConversion)
13694 return SDValue();
13695 // If we are converting to 32-bit integers, we need to add an FP_ROUND.
13696 // This is not valid if the input was originally double precision. It is
13697 // also not profitable to do unless this is an extending load in which
13698 // case doing this combine will allow us to combine consecutive loads.
13699 if (Is32Bit && !isFPExtLoad(NextOp.getOperand(0).getOperand(0)))
13700 return SDValue();
13701 if (N->getOperand(i) != FirstInput)
13702 IsSplat = false;
13703 }
13704
13705 // If this is a splat, we leave it as-is since there will be only a single
13706 // fp-to-int conversion followed by a splat of the integer. This is better
13707 // for 32-bit and smaller ints and neutral for 64-bit ints.
13708 if (IsSplat)
13709 return SDValue();
13710
13711 // Now that we know we have the right type of node, get its operands
13712 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
13713 SDValue In = N->getOperand(i).getOperand(0);
13714 if (Is32Bit) {
13715 // For 32-bit values, we need to add an FP_ROUND node (if we made it
13716 // here, we know that all inputs are extending loads so this is safe).
13717 if (In.isUndef())
13718 Ops.push_back(DAG.getUNDEF(SrcVT));
13719 else {
13720 SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl,
13721 MVT::f32, In.getOperand(0),
13722 DAG.getIntPtrConstant(1, dl));
13723 Ops.push_back(Trunc);
13724 }
13725 } else
13726 Ops.push_back(In.isUndef() ? DAG.getUNDEF(SrcVT) : In.getOperand(0));
13727 }
13728
13729 unsigned Opcode;
13730 if (FirstConversion == PPCISD::FCTIDZ ||
13731 FirstConversion == PPCISD::FCTIWZ)
13732 Opcode = ISD::FP_TO_SINT;
13733 else
13734 Opcode = ISD::FP_TO_UINT;
13735
13736 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32;
13737 SDValue BV = DAG.getBuildVector(NewVT, dl, Ops);
13738 return DAG.getNode(Opcode, dl, TargetVT, BV);
13739 }
13740 return SDValue();
13741}
13742
13743/// Reduce the number of loads when building a vector.
13744///
13745/// Building a vector out of multiple loads can be converted to a load
13746/// of the vector type if the loads are consecutive. If the loads are
13747/// consecutive but in descending order, a shuffle is added at the end
13748/// to reorder the vector.
13749static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
13750 assert(N->getOpcode() == ISD::BUILD_VECTOR &&(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "Should be called with a BUILD_VECTOR node") ? void
(0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"Should be called with a BUILD_VECTOR node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13751, __extension__ __PRETTY_FUNCTION__))
13751 "Should be called with a BUILD_VECTOR node")(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "Should be called with a BUILD_VECTOR node") ? void
(0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"Should be called with a BUILD_VECTOR node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13751, __extension__ __PRETTY_FUNCTION__))
;
13752
13753 SDLoc dl(N);
13754
13755 // Return early for non byte-sized type, as they can't be consecutive.
13756 if (!N->getValueType(0).getVectorElementType().isByteSized())
13757 return SDValue();
13758
13759 bool InputsAreConsecutiveLoads = true;
13760 bool InputsAreReverseConsecutive = true;
13761 unsigned ElemSize = N->getValueType(0).getScalarType().getStoreSize();
13762 SDValue FirstInput = N->getOperand(0);
13763 bool IsRoundOfExtLoad = false;
13764
13765 if (FirstInput.getOpcode() == ISD::FP_ROUND &&
13766 FirstInput.getOperand(0).getOpcode() == ISD::LOAD) {
13767 LoadSDNode *LD = dyn_cast<LoadSDNode>(FirstInput.getOperand(0));
13768 IsRoundOfExtLoad = LD->getExtensionType() == ISD::EXTLOAD;
13769 }
13770 // Not a build vector of (possibly fp_rounded) loads.
13771 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) ||
13772 N->getNumOperands() == 1)
13773 return SDValue();
13774
13775 for (int i = 1, e = N->getNumOperands(); i < e; ++i) {
13776 // If any inputs are fp_round(extload), they all must be.
13777 if (IsRoundOfExtLoad && N->getOperand(i).getOpcode() != ISD::FP_ROUND)
13778 return SDValue();
13779
13780 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) :
13781 N->getOperand(i);
13782 if (NextInput.getOpcode() != ISD::LOAD)
13783 return SDValue();
13784
13785 SDValue PreviousInput =
13786 IsRoundOfExtLoad ? N->getOperand(i-1).getOperand(0) : N->getOperand(i-1);
13787 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(PreviousInput);
13788 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(NextInput);
13789
13790 // If any inputs are fp_round(extload), they all must be.
13791 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD)
13792 return SDValue();
13793
13794 if (!isConsecutiveLS(LD2, LD1, ElemSize, 1, DAG))
13795 InputsAreConsecutiveLoads = false;
13796 if (!isConsecutiveLS(LD1, LD2, ElemSize, 1, DAG))
13797 InputsAreReverseConsecutive = false;
13798
13799 // Exit early if the loads are neither consecutive nor reverse consecutive.
13800 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive)
13801 return SDValue();
13802 }
13803
13804 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) &&(static_cast <bool> (!(InputsAreConsecutiveLoads &&
InputsAreReverseConsecutive) && "The loads cannot be both consecutive and reverse consecutive."
) ? void (0) : __assert_fail ("!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && \"The loads cannot be both consecutive and reverse consecutive.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13805, __extension__ __PRETTY_FUNCTION__))
13805 "The loads cannot be both consecutive and reverse consecutive.")(static_cast <bool> (!(InputsAreConsecutiveLoads &&
InputsAreReverseConsecutive) && "The loads cannot be both consecutive and reverse consecutive."
) ? void (0) : __assert_fail ("!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && \"The loads cannot be both consecutive and reverse consecutive.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13805, __extension__ __PRETTY_FUNCTION__))
;
13806
13807 SDValue FirstLoadOp =
13808 IsRoundOfExtLoad ? FirstInput.getOperand(0) : FirstInput;
13809 SDValue LastLoadOp =
13810 IsRoundOfExtLoad ? N->getOperand(N->getNumOperands()-1).getOperand(0) :
13811 N->getOperand(N->getNumOperands()-1);
13812
13813 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(FirstLoadOp);
13814 LoadSDNode *LDL = dyn_cast<LoadSDNode>(LastLoadOp);
13815 if (InputsAreConsecutiveLoads) {
13816 assert(LD1 && "Input needs to be a LoadSDNode.")(static_cast <bool> (LD1 && "Input needs to be a LoadSDNode."
) ? void (0) : __assert_fail ("LD1 && \"Input needs to be a LoadSDNode.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13816, __extension__ __PRETTY_FUNCTION__))
;
13817 return DAG.getLoad(N->getValueType(0), dl, LD1->getChain(),
13818 LD1->getBasePtr(), LD1->getPointerInfo(),
13819 LD1->getAlignment());
13820 }
13821 if (InputsAreReverseConsecutive) {
13822 assert(LDL && "Input needs to be a LoadSDNode.")(static_cast <bool> (LDL && "Input needs to be a LoadSDNode."
) ? void (0) : __assert_fail ("LDL && \"Input needs to be a LoadSDNode.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 13822, __extension__ __PRETTY_FUNCTION__))
;
13823 SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(),
13824 LDL->getBasePtr(), LDL->getPointerInfo(),
13825 LDL->getAlignment());
13826 SmallVector<int, 16> Ops;
13827 for (int i = N->getNumOperands() - 1; i >= 0; i--)
13828 Ops.push_back(i);
13829
13830 return DAG.getVectorShuffle(N->getValueType(0), dl, Load,
13831 DAG.getUNDEF(N->getValueType(0)), Ops);
13832 }
13833 return SDValue();
13834}
13835
13836// This function adds the required vector_shuffle needed to get
13837// the elements of the vector extract in the correct position
13838// as specified by the CorrectElems encoding.
13839static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG,
13840 SDValue Input, uint64_t Elems,
13841 uint64_t CorrectElems) {
13842 SDLoc dl(N);
13843
13844 unsigned NumElems = Input.getValueType().getVectorNumElements();
13845 SmallVector<int, 16> ShuffleMask(NumElems, -1);
13846
13847 // Knowing the element indices being extracted from the original
13848 // vector and the order in which they're being inserted, just put
13849 // them at element indices required for the instruction.
13850 for (unsigned i = 0; i < N->getNumOperands(); i++) {
13851 if (DAG.getDataLayout().isLittleEndian())
13852 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF;
13853 else
13854 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4;
13855 CorrectElems = CorrectElems >> 8;
13856 Elems = Elems >> 8;
13857 }
13858
13859 SDValue Shuffle =
13860 DAG.getVectorShuffle(Input.getValueType(), dl, Input,
13861 DAG.getUNDEF(Input.getValueType()), ShuffleMask);
13862
13863 EVT VT = N->getValueType(0);
13864 SDValue Conv = DAG.getBitcast(VT, Shuffle);
13865
13866 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
13867 Input.getValueType().getVectorElementType(),
13868 VT.getVectorNumElements());
13869 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Conv,
13870 DAG.getValueType(ExtVT));
13871}
13872
13873// Look for build vector patterns where input operands come from sign
13874// extended vector_extract elements of specific indices. If the correct indices
13875// aren't used, add a vector shuffle to fix up the indices and create
13876// SIGN_EXTEND_INREG node which selects the vector sign extend instructions
13877// during instruction selection.
13878static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
13879 // This array encodes the indices that the vector sign extend instructions
13880 // extract from when extending from one type to another for both BE and LE.
13881 // The right nibble of each byte corresponds to the LE incides.
13882 // and the left nibble of each byte corresponds to the BE incides.
13883 // For example: 0x3074B8FC byte->word
13884 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC
13885 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF
13886 // For example: 0x000070F8 byte->double word
13887 // For LE: the allowed indices are: 0x0,0x8
13888 // For BE: the allowed indices are: 0x7,0xF
13889 uint64_t TargetElems[] = {
13890 0x3074B8FC, // b->w
13891 0x000070F8, // b->d
13892 0x10325476, // h->w
13893 0x00003074, // h->d
13894 0x00001032, // w->d
13895 };
13896
13897 uint64_t Elems = 0;
13898 int Index;
13899 SDValue Input;
13900
13901 auto isSExtOfVecExtract = [&](SDValue Op) -> bool {
13902 if (!Op)
13903 return false;
13904 if (Op.getOpcode() != ISD::SIGN_EXTEND &&
13905 Op.getOpcode() != ISD::SIGN_EXTEND_INREG)
13906 return false;
13907
13908 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value
13909 // of the right width.
13910 SDValue Extract = Op.getOperand(0);
13911 if (Extract.getOpcode() == ISD::ANY_EXTEND)
13912 Extract = Extract.getOperand(0);
13913 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
13914 return false;
13915
13916 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
13917 if (!ExtOp)
13918 return false;
13919
13920 Index = ExtOp->getZExtValue();
13921 if (Input && Input != Extract.getOperand(0))
13922 return false;
13923
13924 if (!Input)
13925 Input = Extract.getOperand(0);
13926
13927 Elems = Elems << 8;
13928 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4;
13929 Elems |= Index;
13930
13931 return true;
13932 };
13933
13934 // If the build vector operands aren't sign extended vector extracts,
13935 // of the same input vector, then return.
13936 for (unsigned i = 0; i < N->getNumOperands(); i++) {
13937 if (!isSExtOfVecExtract(N->getOperand(i))) {
13938 return SDValue();
13939 }
13940 }
13941
13942 // If the vector extract indicies are not correct, add the appropriate
13943 // vector_shuffle.
13944 int TgtElemArrayIdx;
13945 int InputSize = Input.getValueType().getScalarSizeInBits();
13946 int OutputSize = N->getValueType(0).getScalarSizeInBits();
13947 if (InputSize + OutputSize == 40)
13948 TgtElemArrayIdx = 0;
13949 else if (InputSize + OutputSize == 72)
13950 TgtElemArrayIdx = 1;
13951 else if (InputSize + OutputSize == 48)
13952 TgtElemArrayIdx = 2;
13953 else if (InputSize + OutputSize == 80)
13954 TgtElemArrayIdx = 3;
13955 else if (InputSize + OutputSize == 96)
13956 TgtElemArrayIdx = 4;
13957 else
13958 return SDValue();
13959
13960 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx];
13961 CorrectElems = DAG.getDataLayout().isLittleEndian()
13962 ? CorrectElems & 0x0F0F0F0F0F0F0F0F
13963 : CorrectElems & 0xF0F0F0F0F0F0F0F0;
13964 if (Elems != CorrectElems) {
13965 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems);
13966 }
13967
13968 // Regular lowering will catch cases where a shuffle is not needed.
13969 return SDValue();
13970}
13971
13972// Look for the pattern of a load from a narrow width to i128, feeding
13973// into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node
13974// (LXVRZX). This node represents a zero extending load that will be matched
13975// to the Load VSX Vector Rightmost instructions.
13976static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) {
13977 SDLoc DL(N);
13978
13979 // This combine is only eligible for a BUILD_VECTOR of v1i128.
13980 if (N->getValueType(0) != MVT::v1i128)
13981 return SDValue();
13982
13983 SDValue Operand = N->getOperand(0);
13984 // Proceed with the transformation if the operand to the BUILD_VECTOR
13985 // is a load instruction.
13986 if (Operand.getOpcode() != ISD::LOAD)
13987 return SDValue();
13988
13989 auto *LD = cast<LoadSDNode>(Operand);
13990 EVT MemoryType = LD->getMemoryVT();
13991
13992 // This transformation is only valid if the we are loading either a byte,
13993 // halfword, word, or doubleword.
13994 bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 ||
13995 MemoryType == MVT::i32 || MemoryType == MVT::i64;
13996
13997 // Ensure that the load from the narrow width is being zero extended to i128.
13998 if (!ValidLDType ||
13999 (LD->getExtensionType() != ISD::ZEXTLOAD &&
14000 LD->getExtensionType() != ISD::EXTLOAD))
14001 return SDValue();
14002
14003 SDValue LoadOps[] = {
14004 LD->getChain(), LD->getBasePtr(),
14005 DAG.getIntPtrConstant(MemoryType.getScalarSizeInBits(), DL)};
14006
14007 return DAG.getMemIntrinsicNode(PPCISD::LXVRZX, DL,
14008 DAG.getVTList(MVT::v1i128, MVT::Other),
14009 LoadOps, MemoryType, LD->getMemOperand());
14010}
14011
14012SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N,
14013 DAGCombinerInfo &DCI) const {
14014 assert(N->getOpcode() == ISD::BUILD_VECTOR &&(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "Should be called with a BUILD_VECTOR node") ? void
(0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"Should be called with a BUILD_VECTOR node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14015, __extension__ __PRETTY_FUNCTION__))
14015 "Should be called with a BUILD_VECTOR node")(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "Should be called with a BUILD_VECTOR node") ? void
(0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"Should be called with a BUILD_VECTOR node\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14015, __extension__ __PRETTY_FUNCTION__))
;
14016
14017 SelectionDAG &DAG = DCI.DAG;
14018 SDLoc dl(N);
14019
14020 if (!Subtarget.hasVSX())
14021 return SDValue();
14022
14023 // The target independent DAG combiner will leave a build_vector of
14024 // float-to-int conversions intact. We can generate MUCH better code for
14025 // a float-to-int conversion of a vector of floats.
14026 SDValue FirstInput = N->getOperand(0);
14027 if (FirstInput.getOpcode() == PPCISD::MFVSR) {
14028 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI);
14029 if (Reduced)
14030 return Reduced;
14031 }
14032
14033 // If we're building a vector out of consecutive loads, just load that
14034 // vector type.
14035 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG);
14036 if (Reduced)
14037 return Reduced;
14038
14039 // If we're building a vector out of extended elements from another vector
14040 // we have P9 vector integer extend instructions. The code assumes legal
14041 // input types (i.e. it can't handle things like v4i16) so do not run before
14042 // legalization.
14043 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) {
14044 Reduced = combineBVOfVecSExt(N, DAG);
14045 if (Reduced)
14046 return Reduced;
14047 }
14048
14049 // On Power10, the Load VSX Vector Rightmost instructions can be utilized
14050 // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR
14051 // is a load from <valid narrow width> to i128.
14052 if (Subtarget.isISA3_1()) {
14053 SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG);
14054 if (BVOfZLoad)
14055 return BVOfZLoad;
14056 }
14057
14058 if (N->getValueType(0) != MVT::v2f64)
14059 return SDValue();
14060
14061 // Looking for:
14062 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1))
14063 if (FirstInput.getOpcode() != ISD::SINT_TO_FP &&
14064 FirstInput.getOpcode() != ISD::UINT_TO_FP)
14065 return SDValue();
14066 if (N->getOperand(1).getOpcode() != ISD::SINT_TO_FP &&
14067 N->getOperand(1).getOpcode() != ISD::UINT_TO_FP)
14068 return SDValue();
14069 if (FirstInput.getOpcode() != N->getOperand(1).getOpcode())
14070 return SDValue();
14071
14072 SDValue Ext1 = FirstInput.getOperand(0);
14073 SDValue Ext2 = N->getOperand(1).getOperand(0);
14074 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14075 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
14076 return SDValue();
14077
14078 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Ext1.getOperand(1));
14079 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Ext2.getOperand(1));
14080 if (!Ext1Op || !Ext2Op)
14081 return SDValue();
14082 if (Ext1.getOperand(0).getValueType() != MVT::v4i32 ||
14083 Ext1.getOperand(0) != Ext2.getOperand(0))
14084 return SDValue();
14085
14086 int FirstElem = Ext1Op->getZExtValue();
14087 int SecondElem = Ext2Op->getZExtValue();
14088 int SubvecIdx;
14089 if (FirstElem == 0 && SecondElem == 1)
14090 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0;
14091 else if (FirstElem == 2 && SecondElem == 3)
14092 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1;
14093 else
14094 return SDValue();
14095
14096 SDValue SrcVec = Ext1.getOperand(0);
14097 auto NodeType = (N->getOperand(1).getOpcode() == ISD::SINT_TO_FP) ?
14098 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP;
14099 return DAG.getNode(NodeType, dl, MVT::v2f64,
14100 SrcVec, DAG.getIntPtrConstant(SubvecIdx, dl));
14101}
14102
14103SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
14104 DAGCombinerInfo &DCI) const {
14105 assert((N->getOpcode() == ISD::SINT_TO_FP ||(static_cast <bool> ((N->getOpcode() == ISD::SINT_TO_FP
|| N->getOpcode() == ISD::UINT_TO_FP) && "Need an int -> FP conversion node here"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SINT_TO_FP || N->getOpcode() == ISD::UINT_TO_FP) && \"Need an int -> FP conversion node here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14107, __extension__ __PRETTY_FUNCTION__))
14106 N->getOpcode() == ISD::UINT_TO_FP) &&(static_cast <bool> ((N->getOpcode() == ISD::SINT_TO_FP
|| N->getOpcode() == ISD::UINT_TO_FP) && "Need an int -> FP conversion node here"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SINT_TO_FP || N->getOpcode() == ISD::UINT_TO_FP) && \"Need an int -> FP conversion node here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14107, __extension__ __PRETTY_FUNCTION__))
14107 "Need an int -> FP conversion node here")(static_cast <bool> ((N->getOpcode() == ISD::SINT_TO_FP
|| N->getOpcode() == ISD::UINT_TO_FP) && "Need an int -> FP conversion node here"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SINT_TO_FP || N->getOpcode() == ISD::UINT_TO_FP) && \"Need an int -> FP conversion node here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14107, __extension__ __PRETTY_FUNCTION__))
;
14108
14109 if (useSoftFloat() || !Subtarget.has64BitSupport())
14110 return SDValue();
14111
14112 SelectionDAG &DAG = DCI.DAG;
14113 SDLoc dl(N);
14114 SDValue Op(N, 0);
14115
14116 // Don't handle ppc_fp128 here or conversions that are out-of-range capable
14117 // from the hardware.
14118 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
14119 return SDValue();
14120 if (!Op.getOperand(0).getValueType().isSimple())
14121 return SDValue();
14122 if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) ||
14123 Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64))
14124 return SDValue();
14125
14126 SDValue FirstOperand(Op.getOperand(0));
14127 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD &&
14128 (FirstOperand.getValueType() == MVT::i8 ||
14129 FirstOperand.getValueType() == MVT::i16);
14130 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) {
14131 bool Signed = N->getOpcode() == ISD::SINT_TO_FP;
14132 bool DstDouble = Op.getValueType() == MVT::f64;
14133 unsigned ConvOp = Signed ?
14134 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) :
14135 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS);
14136 SDValue WidthConst =
14137 DAG.getIntPtrConstant(FirstOperand.getValueType() == MVT::i8 ? 1 : 2,
14138 dl, false);
14139 LoadSDNode *LDN = cast<LoadSDNode>(FirstOperand.getNode());
14140 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst };
14141 SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl,
14142 DAG.getVTList(MVT::f64, MVT::Other),
14143 Ops, MVT::i8, LDN->getMemOperand());
14144
14145 // For signed conversion, we need to sign-extend the value in the VSR
14146 if (Signed) {
14147 SDValue ExtOps[] = { Ld, WidthConst };
14148 SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps);
14149 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ext);
14150 } else
14151 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ld);
14152 }
14153
14154
14155 // For i32 intermediate values, unfortunately, the conversion functions
14156 // leave the upper 32 bits of the value are undefined. Within the set of
14157 // scalar instructions, we have no method for zero- or sign-extending the
14158 // value. Thus, we cannot handle i32 intermediate values here.
14159 if (Op.getOperand(0).getValueType() == MVT::i32)
14160 return SDValue();
14161
14162 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&(static_cast <bool> ((Op.getOpcode() == ISD::SINT_TO_FP
|| Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"
) ? void (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && \"UINT_TO_FP is supported only with FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14163, __extension__ __PRETTY_FUNCTION__))
14163 "UINT_TO_FP is supported only with FPCVT")(static_cast <bool> ((Op.getOpcode() == ISD::SINT_TO_FP
|| Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"
) ? void (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && \"UINT_TO_FP is supported only with FPCVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14163, __extension__ __PRETTY_FUNCTION__))
;
14164
14165 // If we have FCFIDS, then use it when converting to single-precision.
14166 // Otherwise, convert to double-precision and then round.
14167 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
14168 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
14169 : PPCISD::FCFIDS)
14170 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
14171 : PPCISD::FCFID);
14172 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
14173 ? MVT::f32
14174 : MVT::f64;
14175
14176 // If we're converting from a float, to an int, and back to a float again,
14177 // then we don't need the store/load pair at all.
14178 if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT &&
14179 Subtarget.hasFPCVT()) ||
14180 (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) {
14181 SDValue Src = Op.getOperand(0).getOperand(0);
14182 if (Src.getValueType() == MVT::f32) {
14183 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
14184 DCI.AddToWorklist(Src.getNode());
14185 } else if (Src.getValueType() != MVT::f64) {
14186 // Make sure that we don't pick up a ppc_fp128 source value.
14187 return SDValue();
14188 }
14189
14190 unsigned FCTOp =
14191 Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
14192 PPCISD::FCTIDUZ;
14193
14194 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src);
14195 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp);
14196
14197 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
14198 FP = DAG.getNode(ISD::FP_ROUND, dl,
14199 MVT::f32, FP, DAG.getIntPtrConstant(0, dl));
14200 DCI.AddToWorklist(FP.getNode());
14201 }
14202
14203 return FP;
14204 }
14205
14206 return SDValue();
14207}
14208
14209// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for
14210// builtins) into loads with swaps.
14211SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
14212 DAGCombinerInfo &DCI) const {
14213 SelectionDAG &DAG = DCI.DAG;
14214 SDLoc dl(N);
14215 SDValue Chain;
14216 SDValue Base;
14217 MachineMemOperand *MMO;
14218
14219 switch (N->getOpcode()) {
14220 default:
14221 llvm_unreachable("Unexpected opcode for little endian VSX load")::llvm::llvm_unreachable_internal("Unexpected opcode for little endian VSX load"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14221)
;
14222 case ISD::LOAD: {
14223 LoadSDNode *LD = cast<LoadSDNode>(N);
14224 Chain = LD->getChain();
14225 Base = LD->getBasePtr();
14226 MMO = LD->getMemOperand();
14227 // If the MMO suggests this isn't a load of a full vector, leave
14228 // things alone. For a built-in, we have to make the change for
14229 // correctness, so if there is a size problem that will be a bug.
14230 if (MMO->getSize() < 16)
14231 return SDValue();
14232 break;
14233 }
14234 case ISD::INTRINSIC_W_CHAIN: {
14235 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N);
14236 Chain = Intrin->getChain();
14237 // Similarly to the store case below, Intrin->getBasePtr() doesn't get
14238 // us what we want. Get operand 2 instead.
14239 Base = Intrin->getOperand(2);
14240 MMO = Intrin->getMemOperand();
14241 break;
14242 }
14243 }
14244
14245 MVT VecTy = N->getValueType(0).getSimpleVT();
14246
14247 // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is
14248 // aligned and the type is a vector with elements up to 4 bytes
14249 if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) &&
14250 VecTy.getScalarSizeInBits() <= 32) {
14251 return SDValue();
14252 }
14253
14254 SDValue LoadOps[] = { Chain, Base };
14255 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl,
14256 DAG.getVTList(MVT::v2f64, MVT::Other),
14257 LoadOps, MVT::v2f64, MMO);
14258
14259 DCI.AddToWorklist(Load.getNode());
14260 Chain = Load.getValue(1);
14261 SDValue Swap = DAG.getNode(
14262 PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Load);
14263 DCI.AddToWorklist(Swap.getNode());
14264
14265 // Add a bitcast if the resulting load type doesn't match v2f64.
14266 if (VecTy != MVT::v2f64) {
14267 SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap);
14268 DCI.AddToWorklist(N.getNode());
14269 // Package {bitcast value, swap's chain} to match Load's shape.
14270 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VecTy, MVT::Other),
14271 N, Swap.getValue(1));
14272 }
14273
14274 return Swap;
14275}
14276
14277// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for
14278// builtins) into stores with swaps.
14279SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
14280 DAGCombinerInfo &DCI) const {
14281 SelectionDAG &DAG = DCI.DAG;
14282 SDLoc dl(N);
14283 SDValue Chain;
14284 SDValue Base;
14285 unsigned SrcOpnd;
14286 MachineMemOperand *MMO;
14287
14288 switch (N->getOpcode()) {
14289 default:
14290 llvm_unreachable("Unexpected opcode for little endian VSX store")::llvm::llvm_unreachable_internal("Unexpected opcode for little endian VSX store"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14290)
;
14291 case ISD::STORE: {
14292 StoreSDNode *ST = cast<StoreSDNode>(N);
14293 Chain = ST->getChain();
14294 Base = ST->getBasePtr();
14295 MMO = ST->getMemOperand();
14296 SrcOpnd = 1;
14297 // If the MMO suggests this isn't a store of a full vector, leave
14298 // things alone. For a built-in, we have to make the change for
14299 // correctness, so if there is a size problem that will be a bug.
14300 if (MMO->getSize() < 16)
14301 return SDValue();
14302 break;
14303 }
14304 case ISD::INTRINSIC_VOID: {
14305 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N);
14306 Chain = Intrin->getChain();
14307 // Intrin->getBasePtr() oddly does not get what we want.
14308 Base = Intrin->getOperand(3);
14309 MMO = Intrin->getMemOperand();
14310 SrcOpnd = 2;
14311 break;
14312 }
14313 }
14314
14315 SDValue Src = N->getOperand(SrcOpnd);
14316 MVT VecTy = Src.getValueType().getSimpleVT();
14317
14318 // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is
14319 // aligned and the type is a vector with elements up to 4 bytes
14320 if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) &&
14321 VecTy.getScalarSizeInBits() <= 32) {
14322 return SDValue();
14323 }
14324
14325 // All stores are done as v2f64 and possible bit cast.
14326 if (VecTy != MVT::v2f64) {
14327 Src = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Src);
14328 DCI.AddToWorklist(Src.getNode());
14329 }
14330
14331 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl,
14332 DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Src);
14333 DCI.AddToWorklist(Swap.getNode());
14334 Chain = Swap.getValue(1);
14335 SDValue StoreOps[] = { Chain, Swap, Base };
14336 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl,
14337 DAG.getVTList(MVT::Other),
14338 StoreOps, VecTy, MMO);
14339 DCI.AddToWorklist(Store.getNode());
14340 return Store;
14341}
14342
14343// Handle DAG combine for STORE (FP_TO_INT F).
14344SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
14345 DAGCombinerInfo &DCI) const {
14346
14347 SelectionDAG &DAG = DCI.DAG;
14348 SDLoc dl(N);
14349 unsigned Opcode = N->getOperand(1).getOpcode();
14350
14351 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT)(static_cast <bool> ((Opcode == ISD::FP_TO_SINT || Opcode
== ISD::FP_TO_UINT) && "Not a FP_TO_INT Instruction!"
) ? void (0) : __assert_fail ("(Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) && \"Not a FP_TO_INT Instruction!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14352, __extension__ __PRETTY_FUNCTION__))
14352 && "Not a FP_TO_INT Instruction!")(static_cast <bool> ((Opcode == ISD::FP_TO_SINT || Opcode
== ISD::FP_TO_UINT) && "Not a FP_TO_INT Instruction!"
) ? void (0) : __assert_fail ("(Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) && \"Not a FP_TO_INT Instruction!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14352, __extension__ __PRETTY_FUNCTION__))
;
14353
14354 SDValue Val = N->getOperand(1).getOperand(0);
14355 EVT Op1VT = N->getOperand(1).getValueType();
14356 EVT ResVT = Val.getValueType();
14357
14358 if (!isTypeLegal(ResVT))
14359 return SDValue();
14360
14361 // Only perform combine for conversion to i64/i32 or power9 i16/i8.
14362 bool ValidTypeForStoreFltAsInt =
14363 (Op1VT == MVT::i32 || Op1VT == MVT::i64 ||
14364 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8)));
14365
14366 if (ResVT == MVT::f128 && !Subtarget.hasP9Vector())
14367 return SDValue();
14368
14369 if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Vector() ||
14370 cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt)
14371 return SDValue();
14372
14373 // Extend f32 values to f64
14374 if (ResVT.getScalarSizeInBits() == 32) {
14375 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val);
14376 DCI.AddToWorklist(Val.getNode());
14377 }
14378
14379 // Set signed or unsigned conversion opcode.
14380 unsigned ConvOpcode = (Opcode == ISD::FP_TO_SINT) ?
14381 PPCISD::FP_TO_SINT_IN_VSR :
14382 PPCISD::FP_TO_UINT_IN_VSR;
14383
14384 Val = DAG.getNode(ConvOpcode,
14385 dl, ResVT == MVT::f128 ? MVT::f128 : MVT::f64, Val);
14386 DCI.AddToWorklist(Val.getNode());
14387
14388 // Set number of bytes being converted.
14389 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8;
14390 SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2),
14391 DAG.getIntPtrConstant(ByteSize, dl, false),
14392 DAG.getValueType(Op1VT) };
14393
14394 Val = DAG.getMemIntrinsicNode(PPCISD::ST_VSR_SCAL_INT, dl,
14395 DAG.getVTList(MVT::Other), Ops,
14396 cast<StoreSDNode>(N)->getMemoryVT(),
14397 cast<StoreSDNode>(N)->getMemOperand());
14398
14399 DCI.AddToWorklist(Val.getNode());
14400 return Val;
14401}
14402
14403static bool isAlternatingShuffMask(const ArrayRef<int> &Mask, int NumElts) {
14404 // Check that the source of the element keeps flipping
14405 // (i.e. Mask[i] < NumElts -> Mask[i+i] >= NumElts).
14406 bool PrevElemFromFirstVec = Mask[0] < NumElts;
14407 for (int i = 1, e = Mask.size(); i < e; i++) {
14408 if (PrevElemFromFirstVec && Mask[i] < NumElts)
14409 return false;
14410 if (!PrevElemFromFirstVec && Mask[i] >= NumElts)
14411 return false;
14412 PrevElemFromFirstVec = !PrevElemFromFirstVec;
14413 }
14414 return true;
14415}
14416
14417static bool isSplatBV(SDValue Op) {
14418 if (Op.getOpcode() != ISD::BUILD_VECTOR)
14419 return false;
14420 SDValue FirstOp;
14421
14422 // Find first non-undef input.
14423 for (int i = 0, e = Op.getNumOperands(); i < e; i++) {
14424 FirstOp = Op.getOperand(i);
14425 if (!FirstOp.isUndef())
14426 break;
14427 }
14428
14429 // All inputs are undef or the same as the first non-undef input.
14430 for (int i = 1, e = Op.getNumOperands(); i < e; i++)
14431 if (Op.getOperand(i) != FirstOp && !Op.getOperand(i).isUndef())
14432 return false;
14433 return true;
14434}
14435
14436static SDValue isScalarToVec(SDValue Op) {
14437 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
14438 return Op;
14439 if (Op.getOpcode() != ISD::BITCAST)
14440 return SDValue();
14441 Op = Op.getOperand(0);
14442 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
14443 return Op;
14444 return SDValue();
14445}
14446
14447// Fix up the shuffle mask to account for the fact that the result of
14448// scalar_to_vector is not in lane zero. This just takes all values in
14449// the ranges specified by the min/max indices and adds the number of
14450// elements required to ensure each element comes from the respective
14451// position in the valid lane.
14452// On little endian, that's just the corresponding element in the other
14453// half of the vector. On big endian, it is in the same half but right
14454// justified rather than left justified in that half.
14455static void fixupShuffleMaskForPermutedSToV(SmallVectorImpl<int> &ShuffV,
14456 int LHSMaxIdx, int RHSMinIdx,
14457 int RHSMaxIdx, int HalfVec,
14458 unsigned ValidLaneWidth,
14459 const PPCSubtarget &Subtarget) {
14460 for (int i = 0, e = ShuffV.size(); i < e; i++) {
14461 int Idx = ShuffV[i];
14462 if ((Idx >= 0 && Idx < LHSMaxIdx) || (Idx >= RHSMinIdx && Idx < RHSMaxIdx))
14463 ShuffV[i] +=
14464 Subtarget.isLittleEndian() ? HalfVec : HalfVec - ValidLaneWidth;
14465 }
14466}
14467
14468// Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if
14469// the original is:
14470// (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C))))
14471// In such a case, just change the shuffle mask to extract the element
14472// from the permuted index.
14473static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG,
14474 const PPCSubtarget &Subtarget) {
14475 SDLoc dl(OrigSToV);
14476 EVT VT = OrigSToV.getValueType();
14477 assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR &&(static_cast <bool> (OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR
&& "Expecting a SCALAR_TO_VECTOR here") ? void (0) :
__assert_fail ("OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR && \"Expecting a SCALAR_TO_VECTOR here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14478, __extension__ __PRETTY_FUNCTION__))
14478 "Expecting a SCALAR_TO_VECTOR here")(static_cast <bool> (OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR
&& "Expecting a SCALAR_TO_VECTOR here") ? void (0) :
__assert_fail ("OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR && \"Expecting a SCALAR_TO_VECTOR here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14478, __extension__ __PRETTY_FUNCTION__))
;
14479 SDValue Input = OrigSToV.getOperand(0);
14480
14481 if (Input.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
14482 ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Input.getOperand(1));
14483 SDValue OrigVector = Input.getOperand(0);
14484
14485 // Can't handle non-const element indices or different vector types
14486 // for the input to the extract and the output of the scalar_to_vector.
14487 if (Idx && VT == OrigVector.getValueType()) {
14488 unsigned NumElts = VT.getVectorNumElements();
14489 assert((static_cast <bool> (NumElts > 1 && "Cannot produce a permuted scalar_to_vector for one element vector"
) ? void (0) : __assert_fail ("NumElts > 1 && \"Cannot produce a permuted scalar_to_vector for one element vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14491, __extension__ __PRETTY_FUNCTION__))
14490 NumElts > 1 &&(static_cast <bool> (NumElts > 1 && "Cannot produce a permuted scalar_to_vector for one element vector"
) ? void (0) : __assert_fail ("NumElts > 1 && \"Cannot produce a permuted scalar_to_vector for one element vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14491, __extension__ __PRETTY_FUNCTION__))
14491 "Cannot produce a permuted scalar_to_vector for one element vector")(static_cast <bool> (NumElts > 1 && "Cannot produce a permuted scalar_to_vector for one element vector"
) ? void (0) : __assert_fail ("NumElts > 1 && \"Cannot produce a permuted scalar_to_vector for one element vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14491, __extension__ __PRETTY_FUNCTION__))
;
14492 SmallVector<int, 16> NewMask(NumElts, -1);
14493 unsigned ResultInElt = NumElts / 2;
14494 ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1;
14495 NewMask[ResultInElt] = Idx->getZExtValue();
14496 return DAG.getVectorShuffle(VT, dl, OrigVector, OrigVector, NewMask);
14497 }
14498 }
14499 return DAG.getNode(PPCISD::SCALAR_TO_VECTOR_PERMUTED, dl, VT,
14500 OrigSToV.getOperand(0));
14501}
14502
14503// On little endian subtargets, combine shuffles such as:
14504// vector_shuffle<16,1,17,3,18,5,19,7,20,9,21,11,22,13,23,15>, <zero>, %b
14505// into:
14506// vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7>, <zero>, %b
14507// because the latter can be matched to a single instruction merge.
14508// Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute
14509// to put the value into element zero. Adjust the shuffle mask so that the
14510// vector can remain in permuted form (to prevent a swap prior to a shuffle).
14511// On big endian targets, this is still useful for SCALAR_TO_VECTOR
14512// nodes with elements smaller than doubleword because all the ways
14513// of getting scalar data into a vector register put the value in the
14514// rightmost element of the left half of the vector.
14515SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
14516 SelectionDAG &DAG) const {
14517 SDValue LHS = SVN->getOperand(0);
14518 SDValue RHS = SVN->getOperand(1);
14519 auto Mask = SVN->getMask();
14520 int NumElts = LHS.getValueType().getVectorNumElements();
14521 SDValue Res(SVN, 0);
14522 SDLoc dl(SVN);
14523 bool IsLittleEndian = Subtarget.isLittleEndian();
14524
14525 // On big endian targets this is only useful for subtargets with direct moves.
14526 // On little endian targets it would be useful for all subtargets with VSX.
14527 // However adding special handling for LE subtargets without direct moves
14528 // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8)
14529 // which includes direct moves.
14530 if (!Subtarget.hasDirectMove())
14531 return Res;
14532
14533 // If this is not a shuffle of a shuffle and the first element comes from
14534 // the second vector, canonicalize to the commuted form. This will make it
14535 // more likely to match one of the single instruction patterns.
14536 if (Mask[0] >= NumElts && LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
14537 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) {
14538 std::swap(LHS, RHS);
14539 Res = DAG.getCommutedVectorShuffle(*SVN);
14540 Mask = cast<ShuffleVectorSDNode>(Res)->getMask();
14541 }
14542
14543 // Adjust the shuffle mask if either input vector comes from a
14544 // SCALAR_TO_VECTOR and keep the respective input vector in permuted
14545 // form (to prevent the need for a swap).
14546 SmallVector<int, 16> ShuffV(Mask.begin(), Mask.end());
14547 SDValue SToVLHS = isScalarToVec(LHS);
14548 SDValue SToVRHS = isScalarToVec(RHS);
14549 if (SToVLHS || SToVRHS) {
14550 int NumEltsIn = SToVLHS ? SToVLHS.getValueType().getVectorNumElements()
14551 : SToVRHS.getValueType().getVectorNumElements();
14552 int NumEltsOut = ShuffV.size();
14553 // The width of the "valid lane" (i.e. the lane that contains the value that
14554 // is vectorized) needs to be expressed in terms of the number of elements
14555 // of the shuffle. It is thereby the ratio of the values before and after
14556 // any bitcast.
14557 unsigned ValidLaneWidth =
14558 SToVLHS ? SToVLHS.getValueType().getScalarSizeInBits() /
14559 LHS.getValueType().getScalarSizeInBits()
14560 : SToVRHS.getValueType().getScalarSizeInBits() /
14561 RHS.getValueType().getScalarSizeInBits();
14562
14563 // Initially assume that neither input is permuted. These will be adjusted
14564 // accordingly if either input is.
14565 int LHSMaxIdx = -1;
14566 int RHSMinIdx = -1;
14567 int RHSMaxIdx = -1;
14568 int HalfVec = LHS.getValueType().getVectorNumElements() / 2;
14569
14570 // Get the permuted scalar to vector nodes for the source(s) that come from
14571 // ISD::SCALAR_TO_VECTOR.
14572 // On big endian systems, this only makes sense for element sizes smaller
14573 // than 64 bits since for 64-bit elements, all instructions already put
14574 // the value into element zero. Since scalar size of LHS and RHS may differ
14575 // after isScalarToVec, this should be checked using their own sizes.
14576 if (SToVLHS) {
14577 if (!IsLittleEndian && SToVLHS.getValueType().getScalarSizeInBits() >= 64)
14578 return Res;
14579 // Set up the values for the shuffle vector fixup.
14580 LHSMaxIdx = NumEltsOut / NumEltsIn;
14581 SToVLHS = getSToVPermuted(SToVLHS, DAG, Subtarget);
14582 if (SToVLHS.getValueType() != LHS.getValueType())
14583 SToVLHS = DAG.getBitcast(LHS.getValueType(), SToVLHS);
14584 LHS = SToVLHS;
14585 }
14586 if (SToVRHS) {
14587 if (!IsLittleEndian && SToVRHS.getValueType().getScalarSizeInBits() >= 64)
14588 return Res;
14589 RHSMinIdx = NumEltsOut;
14590 RHSMaxIdx = NumEltsOut / NumEltsIn + RHSMinIdx;
14591 SToVRHS = getSToVPermuted(SToVRHS, DAG, Subtarget);
14592 if (SToVRHS.getValueType() != RHS.getValueType())
14593 SToVRHS = DAG.getBitcast(RHS.getValueType(), SToVRHS);
14594 RHS = SToVRHS;
14595 }
14596
14597 // Fix up the shuffle mask to reflect where the desired element actually is.
14598 // The minimum and maximum indices that correspond to element zero for both
14599 // the LHS and RHS are computed and will control which shuffle mask entries
14600 // are to be changed. For example, if the RHS is permuted, any shuffle mask
14601 // entries in the range [RHSMinIdx,RHSMaxIdx) will be adjusted.
14602 fixupShuffleMaskForPermutedSToV(ShuffV, LHSMaxIdx, RHSMinIdx, RHSMaxIdx,
14603 HalfVec, ValidLaneWidth, Subtarget);
14604 Res = DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV);
14605
14606 // We may have simplified away the shuffle. We won't be able to do anything
14607 // further with it here.
14608 if (!isa<ShuffleVectorSDNode>(Res))
14609 return Res;
14610 Mask = cast<ShuffleVectorSDNode>(Res)->getMask();
14611 }
14612
14613 SDValue TheSplat = IsLittleEndian ? RHS : LHS;
14614 // The common case after we commuted the shuffle is that the RHS is a splat
14615 // and we have elements coming in from the splat at indices that are not
14616 // conducive to using a merge.
14617 // Example:
14618 // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero>
14619 if (!isSplatBV(TheSplat))
14620 return Res;
14621
14622 // We are looking for a mask such that all even elements are from
14623 // one vector and all odd elements from the other.
14624 if (!isAlternatingShuffMask(Mask, NumElts))
14625 return Res;
14626
14627 // Adjust the mask so we are pulling in the same index from the splat
14628 // as the index from the interesting vector in consecutive elements.
14629 if (IsLittleEndian) {
14630 // Example (even elements from first vector):
14631 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero>
14632 if (Mask[0] < NumElts)
14633 for (int i = 1, e = Mask.size(); i < e; i += 2)
14634 ShuffV[i] = (ShuffV[i - 1] + NumElts);
14635 // Example (odd elements from first vector):
14636 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero>
14637 else
14638 for (int i = 0, e = Mask.size(); i < e; i += 2)
14639 ShuffV[i] = (ShuffV[i + 1] + NumElts);
14640 } else {
14641 // Example (even elements from first vector):
14642 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1
14643 if (Mask[0] < NumElts)
14644 for (int i = 0, e = Mask.size(); i < e; i += 2)
14645 ShuffV[i] = ShuffV[i + 1] - NumElts;
14646 // Example (odd elements from first vector):
14647 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1
14648 else
14649 for (int i = 1, e = Mask.size(); i < e; i += 2)
14650 ShuffV[i] = ShuffV[i - 1] - NumElts;
14651 }
14652
14653 // If the RHS has undefs, we need to remove them since we may have created
14654 // a shuffle that adds those instead of the splat value.
14655 SDValue SplatVal =
14656 cast<BuildVectorSDNode>(TheSplat.getNode())->getSplatValue();
14657 TheSplat = DAG.getSplatBuildVector(TheSplat.getValueType(), dl, SplatVal);
14658
14659 if (IsLittleEndian)
14660 RHS = TheSplat;
14661 else
14662 LHS = TheSplat;
14663 return DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV);
14664}
14665
14666SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN,
14667 LSBaseSDNode *LSBase,
14668 DAGCombinerInfo &DCI) const {
14669 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) &&(static_cast <bool> ((ISD::isNormalLoad(LSBase) || ISD::
isNormalStore(LSBase)) && "Not a reverse memop pattern!"
) ? void (0) : __assert_fail ("(ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) && \"Not a reverse memop pattern!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14670, __extension__ __PRETTY_FUNCTION__))
14670 "Not a reverse memop pattern!")(static_cast <bool> ((ISD::isNormalLoad(LSBase) || ISD::
isNormalStore(LSBase)) && "Not a reverse memop pattern!"
) ? void (0) : __assert_fail ("(ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) && \"Not a reverse memop pattern!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14670, __extension__ __PRETTY_FUNCTION__))
;
14671
14672 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool {
14673 auto Mask = SVN->getMask();
14674 int i = 0;
14675 auto I = Mask.rbegin();
14676 auto E = Mask.rend();
14677
14678 for (; I != E; ++I) {
14679 if (*I != i)
14680 return false;
14681 i++;
14682 }
14683 return true;
14684 };
14685
14686 SelectionDAG &DAG = DCI.DAG;
14687 EVT VT = SVN->getValueType(0);
14688
14689 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX())
14690 return SDValue();
14691
14692 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order.
14693 // See comment in PPCVSXSwapRemoval.cpp.
14694 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it.
14695 if (!Subtarget.hasP9Vector())
14696 return SDValue();
14697
14698 if(!IsElementReverse(SVN))
14699 return SDValue();
14700
14701 if (LSBase->getOpcode() == ISD::LOAD) {
14702 // If the load return value 0 has more than one user except the
14703 // shufflevector instruction, it is not profitable to replace the
14704 // shufflevector with a reverse load.
14705 for (SDNode::use_iterator UI = LSBase->use_begin(), UE = LSBase->use_end();
14706 UI != UE; ++UI)
14707 if (UI.getUse().getResNo() == 0 && UI->getOpcode() != ISD::VECTOR_SHUFFLE)
14708 return SDValue();
14709
14710 SDLoc dl(LSBase);
14711 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()};
14712 return DAG.getMemIntrinsicNode(
14713 PPCISD::LOAD_VEC_BE, dl, DAG.getVTList(VT, MVT::Other), LoadOps,
14714 LSBase->getMemoryVT(), LSBase->getMemOperand());
14715 }
14716
14717 if (LSBase->getOpcode() == ISD::STORE) {
14718 // If there are other uses of the shuffle, the swap cannot be avoided.
14719 // Forcing the use of an X-Form (since swapped stores only have
14720 // X-Forms) without removing the swap is unprofitable.
14721 if (!SVN->hasOneUse())
14722 return SDValue();
14723
14724 SDLoc dl(LSBase);
14725 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(0),
14726 LSBase->getBasePtr()};
14727 return DAG.getMemIntrinsicNode(
14728 PPCISD::STORE_VEC_BE, dl, DAG.getVTList(MVT::Other), StoreOps,
14729 LSBase->getMemoryVT(), LSBase->getMemOperand());
14730 }
14731
14732 llvm_unreachable("Expected a load or store node here")::llvm::llvm_unreachable_internal("Expected a load or store node here"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14732)
;
14733}
14734
14735SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
14736 DAGCombinerInfo &DCI) const {
14737 SelectionDAG &DAG = DCI.DAG;
14738 SDLoc dl(N);
14739 switch (N->getOpcode()) {
14740 default: break;
14741 case ISD::ADD:
14742 return combineADD(N, DCI);
14743 case ISD::SHL:
14744 return combineSHL(N, DCI);
14745 case ISD::SRA:
14746 return combineSRA(N, DCI);
14747 case ISD::SRL:
14748 return combineSRL(N, DCI);
14749 case ISD::MUL:
14750 return combineMUL(N, DCI);
14751 case ISD::FMA:
14752 case PPCISD::FNMSUB:
14753 return combineFMALike(N, DCI);
14754 case PPCISD::SHL:
14755 if (isNullConstant(N->getOperand(0))) // 0 << V -> 0.
14756 return N->getOperand(0);
14757 break;
14758 case PPCISD::SRL:
14759 if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0.
14760 return N->getOperand(0);
14761 break;
14762 case PPCISD::SRA:
14763 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
14764 if (C->isNullValue() || // 0 >>s V -> 0.
14765 C->isAllOnesValue()) // -1 >>s V -> -1.
14766 return N->getOperand(0);
14767 }
14768 break;
14769 case ISD::SIGN_EXTEND:
14770 case ISD::ZERO_EXTEND:
14771 case ISD::ANY_EXTEND:
14772 return DAGCombineExtBoolTrunc(N, DCI);
14773 case ISD::TRUNCATE:
14774 return combineTRUNCATE(N, DCI);
14775 case ISD::SETCC:
14776 if (SDValue CSCC = combineSetCC(N, DCI))
14777 return CSCC;
14778 LLVM_FALLTHROUGH[[gnu::fallthrough]];
14779 case ISD::SELECT_CC:
14780 return DAGCombineTruncBoolExt(N, DCI);
14781 case ISD::SINT_TO_FP:
14782 case ISD::UINT_TO_FP:
14783 return combineFPToIntToFP(N, DCI);
14784 case ISD::VECTOR_SHUFFLE:
14785 if (ISD::isNormalLoad(N->getOperand(0).getNode())) {
14786 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(N->getOperand(0));
14787 return combineVReverseMemOP(cast<ShuffleVectorSDNode>(N), LSBase, DCI);
14788 }
14789 return combineVectorShuffle(cast<ShuffleVectorSDNode>(N), DCI.DAG);
14790 case ISD::STORE: {
14791
14792 EVT Op1VT = N->getOperand(1).getValueType();
14793 unsigned Opcode = N->getOperand(1).getOpcode();
14794
14795 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) {
14796 SDValue Val= combineStoreFPToInt(N, DCI);
14797 if (Val)
14798 return Val;
14799 }
14800
14801 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) {
14802 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N->getOperand(1));
14803 SDValue Val= combineVReverseMemOP(SVN, cast<LSBaseSDNode>(N), DCI);
14804 if (Val)
14805 return Val;
14806 }
14807
14808 // Turn STORE (BSWAP) -> sthbrx/stwbrx.
14809 if (cast<StoreSDNode>(N)->isUnindexed() && Opcode == ISD::BSWAP &&
14810 N->getOperand(1).getNode()->hasOneUse() &&
14811 (Op1VT == MVT::i32 || Op1VT == MVT::i16 ||
14812 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) {
14813
14814 // STBRX can only handle simple types and it makes no sense to store less
14815 // two bytes in byte-reversed order.
14816 EVT mVT = cast<StoreSDNode>(N)->getMemoryVT();
14817 if (mVT.isExtended() || mVT.getSizeInBits() < 16)
14818 break;
14819
14820 SDValue BSwapOp = N->getOperand(1).getOperand(0);
14821 // Do an any-extend to 32-bits if this is a half-word input.
14822 if (BSwapOp.getValueType() == MVT::i16)
14823 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp);
14824
14825 // If the type of BSWAP operand is wider than stored memory width
14826 // it need to be shifted to the right side before STBRX.
14827 if (Op1VT.bitsGT(mVT)) {
14828 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits();
14829 BSwapOp = DAG.getNode(ISD::SRL, dl, Op1VT, BSwapOp,
14830 DAG.getConstant(Shift, dl, MVT::i32));
14831 // Need to truncate if this is a bswap of i64 stored as i32/i16.
14832 if (Op1VT == MVT::i64)
14833 BSwapOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BSwapOp);
14834 }
14835
14836 SDValue Ops[] = {
14837 N->getOperand(0), BSwapOp, N->getOperand(2), DAG.getValueType(mVT)
14838 };
14839 return
14840 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other),
14841 Ops, cast<StoreSDNode>(N)->getMemoryVT(),
14842 cast<StoreSDNode>(N)->getMemOperand());
14843 }
14844
14845 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0>
14846 // So it can increase the chance of CSE constant construction.
14847 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() &&
14848 isa<ConstantSDNode>(N->getOperand(1)) && Op1VT == MVT::i32) {
14849 // Need to sign-extended to 64-bits to handle negative values.
14850 EVT MemVT = cast<StoreSDNode>(N)->getMemoryVT();
14851 uint64_t Val64 = SignExtend64(N->getConstantOperandVal(1),
14852 MemVT.getSizeInBits());
14853 SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64);
14854
14855 // DAG.getTruncStore() can't be used here because it doesn't accept
14856 // the general (base + offset) addressing mode.
14857 // So we use UpdateNodeOperands and setTruncatingStore instead.
14858 DAG.UpdateNodeOperands(N, N->getOperand(0), Const64, N->getOperand(2),
14859 N->getOperand(3));
14860 cast<StoreSDNode>(N)->setTruncatingStore(true);
14861 return SDValue(N, 0);
14862 }
14863
14864 // For little endian, VSX stores require generating xxswapd/lxvd2x.
14865 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
14866 if (Op1VT.isSimple()) {
14867 MVT StoreVT = Op1VT.getSimpleVT();
14868 if (Subtarget.needsSwapsForVSXMemOps() &&
14869 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 ||
14870 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32))
14871 return expandVSXStoreForLE(N, DCI);
14872 }
14873 break;
14874 }
14875 case ISD::LOAD: {
14876 LoadSDNode *LD = cast<LoadSDNode>(N);
14877 EVT VT = LD->getValueType(0);
14878
14879 // For little endian, VSX loads require generating lxvd2x/xxswapd.
14880 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
14881 if (VT.isSimple()) {
14882 MVT LoadVT = VT.getSimpleVT();
14883 if (Subtarget.needsSwapsForVSXMemOps() &&
14884 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 ||
14885 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32))
14886 return expandVSXLoadForLE(N, DCI);
14887 }
14888
14889 // We sometimes end up with a 64-bit integer load, from which we extract
14890 // two single-precision floating-point numbers. This happens with
14891 // std::complex<float>, and other similar structures, because of the way we
14892 // canonicalize structure copies. However, if we lack direct moves,
14893 // then the final bitcasts from the extracted integer values to the
14894 // floating-point numbers turn into store/load pairs. Even with direct moves,
14895 // just loading the two floating-point numbers is likely better.
14896 auto ReplaceTwoFloatLoad = [&]() {
14897 if (VT != MVT::i64)
14898 return false;
14899
14900 if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
14901 LD->isVolatile())
14902 return false;
14903
14904 // We're looking for a sequence like this:
14905 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64
14906 // t16: i64 = srl t13, Constant:i32<32>
14907 // t17: i32 = truncate t16
14908 // t18: f32 = bitcast t17
14909 // t19: i32 = truncate t13
14910 // t20: f32 = bitcast t19
14911
14912 if (!LD->hasNUsesOfValue(2, 0))
14913 return false;
14914
14915 auto UI = LD->use_begin();
14916 while (UI.getUse().getResNo() != 0) ++UI;
14917 SDNode *Trunc = *UI++;
14918 while (UI.getUse().getResNo() != 0) ++UI;
14919 SDNode *RightShift = *UI;
14920 if (Trunc->getOpcode() != ISD::TRUNCATE)
14921 std::swap(Trunc, RightShift);
14922
14923 if (Trunc->getOpcode() != ISD::TRUNCATE ||
14924 Trunc->getValueType(0) != MVT::i32 ||
14925 !Trunc->hasOneUse())
14926 return false;
14927 if (RightShift->getOpcode() != ISD::SRL ||
14928 !isa<ConstantSDNode>(RightShift->getOperand(1)) ||
14929 RightShift->getConstantOperandVal(1) != 32 ||
14930 !RightShift->hasOneUse())
14931 return false;
14932
14933 SDNode *Trunc2 = *RightShift->use_begin();
14934 if (Trunc2->getOpcode() != ISD::TRUNCATE ||
14935 Trunc2->getValueType(0) != MVT::i32 ||
14936 !Trunc2->hasOneUse())
14937 return false;
14938
14939 SDNode *Bitcast = *Trunc->use_begin();
14940 SDNode *Bitcast2 = *Trunc2->use_begin();
14941
14942 if (Bitcast->getOpcode() != ISD::BITCAST ||
14943 Bitcast->getValueType(0) != MVT::f32)
14944 return false;
14945 if (Bitcast2->getOpcode() != ISD::BITCAST ||
14946 Bitcast2->getValueType(0) != MVT::f32)
14947 return false;
14948
14949 if (Subtarget.isLittleEndian())
14950 std::swap(Bitcast, Bitcast2);
14951
14952 // Bitcast has the second float (in memory-layout order) and Bitcast2
14953 // has the first one.
14954
14955 SDValue BasePtr = LD->getBasePtr();
14956 if (LD->isIndexed()) {
14957 assert(LD->getAddressingMode() == ISD::PRE_INC &&(static_cast <bool> (LD->getAddressingMode() == ISD::
PRE_INC && "Non-pre-inc AM on PPC?") ? void (0) : __assert_fail
("LD->getAddressingMode() == ISD::PRE_INC && \"Non-pre-inc AM on PPC?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14958, __extension__ __PRETTY_FUNCTION__))
14958 "Non-pre-inc AM on PPC?")(static_cast <bool> (LD->getAddressingMode() == ISD::
PRE_INC && "Non-pre-inc AM on PPC?") ? void (0) : __assert_fail
("LD->getAddressingMode() == ISD::PRE_INC && \"Non-pre-inc AM on PPC?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 14958, __extension__ __PRETTY_FUNCTION__))
;
14959 BasePtr =
14960 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
14961 LD->getOffset());
14962 }
14963
14964 auto MMOFlags =
14965 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile;
14966 SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr,
14967 LD->getPointerInfo(), LD->getAlignment(),
14968 MMOFlags, LD->getAAInfo());
14969 SDValue AddPtr =
14970 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
14971 BasePtr, DAG.getIntPtrConstant(4, dl));
14972 SDValue FloatLoad2 = DAG.getLoad(
14973 MVT::f32, dl, SDValue(FloatLoad.getNode(), 1), AddPtr,
14974 LD->getPointerInfo().getWithOffset(4),
14975 MinAlign(LD->getAlignment(), 4), MMOFlags, LD->getAAInfo());
14976
14977 if (LD->isIndexed()) {
14978 // Note that DAGCombine should re-form any pre-increment load(s) from
14979 // what is produced here if that makes sense.
14980 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), BasePtr);
14981 }
14982
14983 DCI.CombineTo(Bitcast2, FloatLoad);
14984 DCI.CombineTo(Bitcast, FloatLoad2);
14985
14986 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, LD->isIndexed() ? 2 : 1),
14987 SDValue(FloatLoad2.getNode(), 1));
14988 return true;
14989 };
14990
14991 if (ReplaceTwoFloatLoad())
14992 return SDValue(N, 0);
14993
14994 EVT MemVT = LD->getMemoryVT();
14995 Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
14996 Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
14997 if (LD->isUnindexed() && VT.isVector() &&
14998 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
14999 // P8 and later hardware should just use LOAD.
15000 !Subtarget.hasP8Vector() &&
15001 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
15002 VT == MVT::v4f32))) &&
15003 LD->getAlign() < ABIAlignment) {
15004 // This is a type-legal unaligned Altivec load.
15005 SDValue Chain = LD->getChain();
15006 SDValue Ptr = LD->getBasePtr();
15007 bool isLittleEndian = Subtarget.isLittleEndian();
15008
15009 // This implements the loading of unaligned vectors as described in
15010 // the venerable Apple Velocity Engine overview. Specifically:
15011 // https://developer.apple.com/hardwaredrivers/ve/alignment.html
15012 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html
15013 //
15014 // The general idea is to expand a sequence of one or more unaligned
15015 // loads into an alignment-based permutation-control instruction (lvsl
15016 // or lvsr), a series of regular vector loads (which always truncate
15017 // their input address to an aligned address), and a series of
15018 // permutations. The results of these permutations are the requested
15019 // loaded values. The trick is that the last "extra" load is not taken
15020 // from the address you might suspect (sizeof(vector) bytes after the
15021 // last requested load), but rather sizeof(vector) - 1 bytes after the
15022 // last requested vector. The point of this is to avoid a page fault if
15023 // the base address happened to be aligned. This works because if the
15024 // base address is aligned, then adding less than a full vector length
15025 // will cause the last vector in the sequence to be (re)loaded.
15026 // Otherwise, the next vector will be fetched as you might suspect was
15027 // necessary.
15028
15029 // We might be able to reuse the permutation generation from
15030 // a different base address offset from this one by an aligned amount.
15031 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this
15032 // optimization later.
15033 Intrinsic::ID Intr, IntrLD, IntrPerm;
15034 MVT PermCntlTy, PermTy, LDTy;
15035 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
15036 : Intrinsic::ppc_altivec_lvsl;
15037 IntrLD = Intrinsic::ppc_altivec_lvx;
15038 IntrPerm = Intrinsic::ppc_altivec_vperm;
15039 PermCntlTy = MVT::v16i8;
15040 PermTy = MVT::v4i32;
15041 LDTy = MVT::v4i32;
15042
15043 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy);
15044
15045 // Create the new MMO for the new base load. It is like the original MMO,
15046 // but represents an area in memory almost twice the vector size centered
15047 // on the original address. If the address is unaligned, we might start
15048 // reading up to (sizeof(vector)-1) bytes below the address of the
15049 // original unaligned load.
15050 MachineFunction &MF = DAG.getMachineFunction();
15051 MachineMemOperand *BaseMMO =
15052 MF.getMachineMemOperand(LD->getMemOperand(),
15053 -(long)MemVT.getStoreSize()+1,
15054 2*MemVT.getStoreSize()-1);
15055
15056 // Create the new base load.
15057 SDValue LDXIntID =
15058 DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout()));
15059 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
15060 SDValue BaseLoad =
15061 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl,
15062 DAG.getVTList(PermTy, MVT::Other),
15063 BaseLoadOps, LDTy, BaseMMO);
15064
15065 // Note that the value of IncOffset (which is provided to the next
15066 // load's pointer info offset value, and thus used to calculate the
15067 // alignment), and the value of IncValue (which is actually used to
15068 // increment the pointer value) are different! This is because we
15069 // require the next load to appear to be aligned, even though it
15070 // is actually offset from the base pointer by a lesser amount.
15071 int IncOffset = VT.getSizeInBits() / 8;
15072 int IncValue = IncOffset;
15073
15074 // Walk (both up and down) the chain looking for another load at the real
15075 // (aligned) offset (the alignment of the other load does not matter in
15076 // this case). If found, then do not use the offset reduction trick, as
15077 // that will prevent the loads from being later combined (as they would
15078 // otherwise be duplicates).
15079 if (!findConsecutiveLoad(LD, DAG))
15080 --IncValue;
15081
15082 SDValue Increment =
15083 DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout()));
15084 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
15085
15086 MachineMemOperand *ExtraMMO =
15087 MF.getMachineMemOperand(LD->getMemOperand(),
15088 1, 2*MemVT.getStoreSize()-1);
15089 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
15090 SDValue ExtraLoad =
15091 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl,
15092 DAG.getVTList(PermTy, MVT::Other),
15093 ExtraLoadOps, LDTy, ExtraMMO);
15094
15095 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
15096 BaseLoad.getValue(1), ExtraLoad.getValue(1));
15097
15098 // Because vperm has a big-endian bias, we must reverse the order
15099 // of the input vectors and complement the permute control vector
15100 // when generating little endian code. We have already handled the
15101 // latter by using lvsr instead of lvsl, so just reverse BaseLoad
15102 // and ExtraLoad here.
15103 SDValue Perm;
15104 if (isLittleEndian)
15105 Perm = BuildIntrinsicOp(IntrPerm,
15106 ExtraLoad, BaseLoad, PermCntl, DAG, dl);
15107 else
15108 Perm = BuildIntrinsicOp(IntrPerm,
15109 BaseLoad, ExtraLoad, PermCntl, DAG, dl);
15110
15111 if (VT != PermTy)
15112 Perm = Subtarget.hasAltivec()
15113 ? DAG.getNode(ISD::BITCAST, dl, VT, Perm)
15114 : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm,
15115 DAG.getTargetConstant(1, dl, MVT::i64));
15116 // second argument is 1 because this rounding
15117 // is always exact.
15118
15119 // The output of the permutation is our loaded result, the TokenFactor is
15120 // our new chain.
15121 DCI.CombineTo(N, Perm, TF);
15122 return SDValue(N, 0);
15123 }
15124 }
15125 break;
15126 case ISD::INTRINSIC_WO_CHAIN: {
15127 bool isLittleEndian = Subtarget.isLittleEndian();
15128 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
15129 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
15130 : Intrinsic::ppc_altivec_lvsl);
15131 if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) {
15132 SDValue Add = N->getOperand(1);
15133
15134 int Bits = 4 /* 16 byte alignment */;
15135
15136 if (DAG.MaskedValueIsZero(Add->getOperand(1),
15137 APInt::getAllOnesValue(Bits /* alignment */)
15138 .zext(Add.getScalarValueSizeInBits()))) {
15139 SDNode *BasePtr = Add->getOperand(0).getNode();
15140 for (SDNode::use_iterator UI = BasePtr->use_begin(),
15141 UE = BasePtr->use_end();
15142 UI != UE; ++UI) {
15143 if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
15144 cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() ==
15145 IID) {
15146 // We've found another LVSL/LVSR, and this address is an aligned
15147 // multiple of that one. The results will be the same, so use the
15148 // one we've just found instead.
15149
15150 return SDValue(*UI, 0);
15151 }
15152 }
15153 }
15154
15155 if (isa<ConstantSDNode>(Add->getOperand(1))) {
15156 SDNode *BasePtr = Add->getOperand(0).getNode();
15157 for (SDNode::use_iterator UI = BasePtr->use_begin(),
15158 UE = BasePtr->use_end(); UI != UE; ++UI) {
15159 if (UI->getOpcode() == ISD::ADD &&
15160 isa<ConstantSDNode>(UI->getOperand(1)) &&
15161 (cast<ConstantSDNode>(Add->getOperand(1))->getZExtValue() -
15162 cast<ConstantSDNode>(UI->getOperand(1))->getZExtValue()) %
15163 (1ULL << Bits) == 0) {
15164 SDNode *OtherAdd = *UI;
15165 for (SDNode::use_iterator VI = OtherAdd->use_begin(),
15166 VE = OtherAdd->use_end(); VI != VE; ++VI) {
15167 if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
15168 cast<ConstantSDNode>(VI->getOperand(0))->getZExtValue() == IID) {
15169 return SDValue(*VI, 0);
15170 }
15171 }
15172 }
15173 }
15174 }
15175 }
15176
15177 // Combine vmaxsw/h/b(a, a's negation) to abs(a)
15178 // Expose the vabsduw/h/b opportunity for down stream
15179 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() &&
15180 (IID == Intrinsic::ppc_altivec_vmaxsw ||
15181 IID == Intrinsic::ppc_altivec_vmaxsh ||
15182 IID == Intrinsic::ppc_altivec_vmaxsb)) {
15183 SDValue V1 = N->getOperand(1);
15184 SDValue V2 = N->getOperand(2);
15185 if ((V1.getSimpleValueType() == MVT::v4i32 ||
15186 V1.getSimpleValueType() == MVT::v8i16 ||
15187 V1.getSimpleValueType() == MVT::v16i8) &&
15188 V1.getSimpleValueType() == V2.getSimpleValueType()) {
15189 // (0-a, a)
15190 if (V1.getOpcode() == ISD::SUB &&
15191 ISD::isBuildVectorAllZeros(V1.getOperand(0).getNode()) &&
15192 V1.getOperand(1) == V2) {
15193 return DAG.getNode(ISD::ABS, dl, V2.getValueType(), V2);
15194 }
15195 // (a, 0-a)
15196 if (V2.getOpcode() == ISD::SUB &&
15197 ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()) &&
15198 V2.getOperand(1) == V1) {
15199 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1);
15200 }
15201 // (x-y, y-x)
15202 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB &&
15203 V1.getOperand(0) == V2.getOperand(1) &&
15204 V1.getOperand(1) == V2.getOperand(0)) {
15205 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1);
15206 }
15207 }
15208 }
15209 }
15210
15211 break;
15212 case ISD::INTRINSIC_W_CHAIN:
15213 // For little endian, VSX loads require generating lxvd2x/xxswapd.
15214 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
15215 if (Subtarget.needsSwapsForVSXMemOps()) {
15216 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
15217 default:
15218 break;
15219 case Intrinsic::ppc_vsx_lxvw4x:
15220 case Intrinsic::ppc_vsx_lxvd2x:
15221 return expandVSXLoadForLE(N, DCI);
15222 }
15223 }
15224 break;
15225 case ISD::INTRINSIC_VOID:
15226 // For little endian, VSX stores require generating xxswapd/stxvd2x.
15227 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
15228 if (Subtarget.needsSwapsForVSXMemOps()) {
15229 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
15230 default:
15231 break;
15232 case Intrinsic::ppc_vsx_stxvw4x:
15233 case Intrinsic::ppc_vsx_stxvd2x:
15234 return expandVSXStoreForLE(N, DCI);
15235 }
15236 }
15237 break;
15238 case ISD::BSWAP: {
15239 // Turn BSWAP (LOAD) -> lhbrx/lwbrx.
15240 // For subtargets without LDBRX, we can still do better than the default
15241 // expansion even for 64-bit BSWAP (LOAD).
15242 bool Is64BitBswapOn64BitTgt =
15243 Subtarget.isPPC64() && N->getValueType(0) == MVT::i64;
15244 bool IsSingleUseNormalLd = ISD::isNormalLoad(N->getOperand(0).getNode()) &&
15245 N->getOperand(0).hasOneUse();
15246 if (IsSingleUseNormalLd &&
15247 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 ||
15248 (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) {
15249 SDValue Load = N->getOperand(0);
15250 LoadSDNode *LD = cast<LoadSDNode>(Load);
15251 // Create the byte-swapping load.
15252 SDValue Ops[] = {
15253 LD->getChain(), // Chain
15254 LD->getBasePtr(), // Ptr
15255 DAG.getValueType(N->getValueType(0)) // VT
15256 };
15257 SDValue BSLoad =
15258 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl,
15259 DAG.getVTList(N->getValueType(0) == MVT::i64 ?
15260 MVT::i64 : MVT::i32, MVT::Other),
15261 Ops, LD->getMemoryVT(), LD->getMemOperand());
15262
15263 // If this is an i16 load, insert the truncate.
15264 SDValue ResVal = BSLoad;
15265 if (N->getValueType(0) == MVT::i16)
15266 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad);
15267
15268 // First, combine the bswap away. This makes the value produced by the
15269 // load dead.
15270 DCI.CombineTo(N, ResVal);
15271
15272 // Next, combine the load away, we give it a bogus result value but a real
15273 // chain result. The result value is dead because the bswap is dead.
15274 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1));
15275
15276 // Return N so it doesn't get rechecked!
15277 return SDValue(N, 0);
15278 }
15279 // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only
15280 // before legalization so that the BUILD_PAIR is handled correctly.
15281 if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt ||
15282 !IsSingleUseNormalLd)
15283 return SDValue();
15284 LoadSDNode *LD = cast<LoadSDNode>(N->getOperand(0));
15285
15286 // Can't split volatile or atomic loads.
15287 if (!LD->isSimple())
15288 return SDValue();
15289 SDValue BasePtr = LD->getBasePtr();
15290 SDValue Lo = DAG.getLoad(MVT::i32, dl, LD->getChain(), BasePtr,
15291 LD->getPointerInfo(), LD->getAlignment());
15292 Lo = DAG.getNode(ISD::BSWAP, dl, MVT::i32, Lo);
15293 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
15294 DAG.getIntPtrConstant(4, dl));
15295 MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand(
15296 LD->getMemOperand(), 4, 4);
15297 SDValue Hi = DAG.getLoad(MVT::i32, dl, LD->getChain(), BasePtr, NewMMO);
15298 Hi = DAG.getNode(ISD::BSWAP, dl, MVT::i32, Hi);
15299 SDValue Res;
15300 if (Subtarget.isLittleEndian())
15301 Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Hi, Lo);
15302 else
15303 Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
15304 SDValue TF =
15305 DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
15306 Hi.getOperand(0).getValue(1), Lo.getOperand(0).getValue(1));
15307 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), TF);
15308 return Res;
15309 }
15310 case PPCISD::VCMP:
15311 // If a VCMP_rec node already exists with exactly the same operands as this
15312 // node, use its result instead of this node (VCMP_rec computes both a CR6
15313 // and a normal output).
15314 //
15315 if (!N->getOperand(0).hasOneUse() &&
15316 !N->getOperand(1).hasOneUse() &&
15317 !N->getOperand(2).hasOneUse()) {
15318
15319 // Scan all of the users of the LHS, looking for VCMP_rec's that match.
15320 SDNode *VCMPrecNode = nullptr;
15321
15322 SDNode *LHSN = N->getOperand(0).getNode();
15323 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end();
15324 UI != E; ++UI)
15325 if (UI->getOpcode() == PPCISD::VCMP_rec &&
15326 UI->getOperand(1) == N->getOperand(1) &&
15327 UI->getOperand(2) == N->getOperand(2) &&
15328 UI->getOperand(0) == N->getOperand(0)) {
15329 VCMPrecNode = *UI;
15330 break;
15331 }
15332
15333 // If there is no VCMP_rec node, or if the flag value has a single use,
15334 // don't transform this.
15335 if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(0, 1))
15336 break;
15337
15338 // Look at the (necessarily single) use of the flag value. If it has a
15339 // chain, this transformation is more complex. Note that multiple things
15340 // could use the value result, which we should ignore.
15341 SDNode *FlagUser = nullptr;
15342 for (SDNode::use_iterator UI = VCMPrecNode->use_begin();
15343 FlagUser == nullptr; ++UI) {
15344 assert(UI != VCMPrecNode->use_end() && "Didn't find user!")(static_cast <bool> (UI != VCMPrecNode->use_end() &&
"Didn't find user!") ? void (0) : __assert_fail ("UI != VCMPrecNode->use_end() && \"Didn't find user!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15344, __extension__ __PRETTY_FUNCTION__))
;
15345 SDNode *User = *UI;
15346 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
15347 if (User->getOperand(i) == SDValue(VCMPrecNode, 1)) {
15348 FlagUser = User;
15349 break;
15350 }
15351 }
15352 }
15353
15354 // If the user is a MFOCRF instruction, we know this is safe.
15355 // Otherwise we give up for right now.
15356 if (FlagUser->getOpcode() == PPCISD::MFOCRF)
15357 return SDValue(VCMPrecNode, 0);
15358 }
15359 break;
15360 case ISD::BRCOND: {
15361 SDValue Cond = N->getOperand(1);
15362 SDValue Target = N->getOperand(2);
15363
15364 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
15365 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() ==
15366 Intrinsic::loop_decrement) {
15367
15368 // We now need to make the intrinsic dead (it cannot be instruction
15369 // selected).
15370 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0));
15371 assert(Cond.getNode()->hasOneUse() &&(static_cast <bool> (Cond.getNode()->hasOneUse() &&
"Counter decrement has more than one use") ? void (0) : __assert_fail
("Cond.getNode()->hasOneUse() && \"Counter decrement has more than one use\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15372, __extension__ __PRETTY_FUNCTION__))
15372 "Counter decrement has more than one use")(static_cast <bool> (Cond.getNode()->hasOneUse() &&
"Counter decrement has more than one use") ? void (0) : __assert_fail
("Cond.getNode()->hasOneUse() && \"Counter decrement has more than one use\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15372, __extension__ __PRETTY_FUNCTION__))
;
15373
15374 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other,
15375 N->getOperand(0), Target);
15376 }
15377 }
15378 break;
15379 case ISD::BR_CC: {
15380 // If this is a branch on an altivec predicate comparison, lower this so
15381 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This
15382 // lowering is done pre-legalize, because the legalizer lowers the predicate
15383 // compare down to code that is difficult to reassemble.
15384 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
15385 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3);
15386
15387 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero
15388 // value. If so, pass-through the AND to get to the intrinsic.
15389 if (LHS.getOpcode() == ISD::AND &&
15390 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN &&
15391 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() ==
15392 Intrinsic::loop_decrement &&
15393 isa<ConstantSDNode>(LHS.getOperand(1)) &&
15394 !isNullConstant(LHS.getOperand(1)))
15395 LHS = LHS.getOperand(0);
15396
15397 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
15398 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() ==
15399 Intrinsic::loop_decrement &&
15400 isa<ConstantSDNode>(RHS)) {
15401 assert((CC == ISD::SETEQ || CC == ISD::SETNE) &&(static_cast <bool> ((CC == ISD::SETEQ || CC == ISD::SETNE
) && "Counter decrement comparison is not EQ or NE") ?
void (0) : __assert_fail ("(CC == ISD::SETEQ || CC == ISD::SETNE) && \"Counter decrement comparison is not EQ or NE\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15402, __extension__ __PRETTY_FUNCTION__))
15402 "Counter decrement comparison is not EQ or NE")(static_cast <bool> ((CC == ISD::SETEQ || CC == ISD::SETNE
) && "Counter decrement comparison is not EQ or NE") ?
void (0) : __assert_fail ("(CC == ISD::SETEQ || CC == ISD::SETNE) && \"Counter decrement comparison is not EQ or NE\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15402, __extension__ __PRETTY_FUNCTION__))
;
15403
15404 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue();
15405 bool isBDNZ = (CC == ISD::SETEQ && Val) ||
15406 (CC == ISD::SETNE && !Val);
15407
15408 // We now need to make the intrinsic dead (it cannot be instruction
15409 // selected).
15410 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0));
15411 assert(LHS.getNode()->hasOneUse() &&(static_cast <bool> (LHS.getNode()->hasOneUse() &&
"Counter decrement has more than one use") ? void (0) : __assert_fail
("LHS.getNode()->hasOneUse() && \"Counter decrement has more than one use\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15412, __extension__ __PRETTY_FUNCTION__))
15412 "Counter decrement has more than one use")(static_cast <bool> (LHS.getNode()->hasOneUse() &&
"Counter decrement has more than one use") ? void (0) : __assert_fail
("LHS.getNode()->hasOneUse() && \"Counter decrement has more than one use\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15412, __extension__ __PRETTY_FUNCTION__))
;
15413
15414 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other,
15415 N->getOperand(0), N->getOperand(4));
15416 }
15417
15418 int CompareOpc;
15419 bool isDot;
15420
15421 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
15422 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
15423 getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) {
15424 assert(isDot && "Can't compare against a vector result!")(static_cast <bool> (isDot && "Can't compare against a vector result!"
) ? void (0) : __assert_fail ("isDot && \"Can't compare against a vector result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15424, __extension__ __PRETTY_FUNCTION__))
;
15425
15426 // If this is a comparison against something other than 0/1, then we know
15427 // that the condition is never/always true.
15428 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue();
15429 if (Val != 0 && Val != 1) {
15430 if (CC == ISD::SETEQ) // Cond never true, remove branch.
15431 return N->getOperand(0);
15432 // Always !=, turn it into an unconditional branch.
15433 return DAG.getNode(ISD::BR, dl, MVT::Other,
15434 N->getOperand(0), N->getOperand(4));
15435 }
15436
15437 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0);
15438
15439 // Create the PPCISD altivec 'dot' comparison node.
15440 SDValue Ops[] = {
15441 LHS.getOperand(2), // LHS of compare
15442 LHS.getOperand(3), // RHS of compare
15443 DAG.getConstant(CompareOpc, dl, MVT::i32)
15444 };
15445 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue };
15446 SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops);
15447
15448 // Unpack the result based on how the target uses it.
15449 PPC::Predicate CompOpc;
15450 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) {
15451 default: // Can't happen, don't crash on invalid number though.
15452 case 0: // Branch on the value of the EQ bit of CR6.
15453 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE;
15454 break;
15455 case 1: // Branch on the inverted value of the EQ bit of CR6.
15456 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ;
15457 break;
15458 case 2: // Branch on the value of the LT bit of CR6.
15459 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE;
15460 break;
15461 case 3: // Branch on the inverted value of the LT bit of CR6.
15462 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT;
15463 break;
15464 }
15465
15466 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0),
15467 DAG.getConstant(CompOpc, dl, MVT::i32),
15468 DAG.getRegister(PPC::CR6, MVT::i32),
15469 N->getOperand(4), CompNode.getValue(1));
15470 }
15471 break;
15472 }
15473 case ISD::BUILD_VECTOR:
15474 return DAGCombineBuildVector(N, DCI);
15475 case ISD::ABS:
15476 return combineABS(N, DCI);
15477 case ISD::VSELECT:
15478 return combineVSelect(N, DCI);
15479 }
15480
15481 return SDValue();
15482}
15483
15484SDValue
15485PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
15486 SelectionDAG &DAG,
15487 SmallVectorImpl<SDNode *> &Created) const {
15488 // fold (sdiv X, pow2)
15489 EVT VT = N->getValueType(0);
15490 if (VT == MVT::i64 && !Subtarget.isPPC64())
15491 return SDValue();
15492 if ((VT != MVT::i32 && VT != MVT::i64) ||
15493 !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()))
15494 return SDValue();
15495
15496 SDLoc DL(N);
15497 SDValue N0 = N->getOperand(0);
15498
15499 bool IsNegPow2 = (-Divisor).isPowerOf2();
15500 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros();
15501 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT);
15502
15503 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt);
15504 Created.push_back(Op.getNode());
15505
15506 if (IsNegPow2) {
15507 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
15508 Created.push_back(Op.getNode());
15509 }
15510
15511 return Op;
15512}
15513
15514//===----------------------------------------------------------------------===//
15515// Inline Assembly Support
15516//===----------------------------------------------------------------------===//
15517
15518void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
15519 KnownBits &Known,
15520 const APInt &DemandedElts,
15521 const SelectionDAG &DAG,
15522 unsigned Depth) const {
15523 Known.resetAll();
15524 switch (Op.getOpcode()) {
15525 default: break;
15526 case PPCISD::LBRX: {
15527 // lhbrx is known to have the top bits cleared out.
15528 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16)
15529 Known.Zero = 0xFFFF0000;
15530 break;
15531 }
15532 case ISD::INTRINSIC_WO_CHAIN: {
15533 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) {
15534 default: break;
15535 case Intrinsic::ppc_altivec_vcmpbfp_p:
15536 case Intrinsic::ppc_altivec_vcmpeqfp_p:
15537 case Intrinsic::ppc_altivec_vcmpequb_p:
15538 case Intrinsic::ppc_altivec_vcmpequh_p:
15539 case Intrinsic::ppc_altivec_vcmpequw_p:
15540 case Intrinsic::ppc_altivec_vcmpequd_p:
15541 case Intrinsic::ppc_altivec_vcmpequq_p:
15542 case Intrinsic::ppc_altivec_vcmpgefp_p:
15543 case Intrinsic::ppc_altivec_vcmpgtfp_p:
15544 case Intrinsic::ppc_altivec_vcmpgtsb_p:
15545 case Intrinsic::ppc_altivec_vcmpgtsh_p:
15546 case Intrinsic::ppc_altivec_vcmpgtsw_p:
15547 case Intrinsic::ppc_altivec_vcmpgtsd_p:
15548 case Intrinsic::ppc_altivec_vcmpgtsq_p:
15549 case Intrinsic::ppc_altivec_vcmpgtub_p:
15550 case Intrinsic::ppc_altivec_vcmpgtuh_p:
15551 case Intrinsic::ppc_altivec_vcmpgtuw_p:
15552 case Intrinsic::ppc_altivec_vcmpgtud_p:
15553 case Intrinsic::ppc_altivec_vcmpgtuq_p:
15554 Known.Zero = ~1U; // All bits but the low one are known to be zero.
15555 break;
15556 }
15557 }
15558 }
15559}
15560
15561Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
15562 switch (Subtarget.getCPUDirective()) {
15563 default: break;
15564 case PPC::DIR_970:
15565 case PPC::DIR_PWR4:
15566 case PPC::DIR_PWR5:
15567 case PPC::DIR_PWR5X:
15568 case PPC::DIR_PWR6:
15569 case PPC::DIR_PWR6X:
15570 case PPC::DIR_PWR7:
15571 case PPC::DIR_PWR8:
15572 case PPC::DIR_PWR9:
15573 case PPC::DIR_PWR10:
15574 case PPC::DIR_PWR_FUTURE: {
15575 if (!ML)
15576 break;
15577
15578 if (!DisableInnermostLoopAlign32) {
15579 // If the nested loop is an innermost loop, prefer to a 32-byte alignment,
15580 // so that we can decrease cache misses and branch-prediction misses.
15581 // Actual alignment of the loop will depend on the hotness check and other
15582 // logic in alignBlocks.
15583 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty())
15584 return Align(32);
15585 }
15586
15587 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
15588
15589 // For small loops (between 5 and 8 instructions), align to a 32-byte
15590 // boundary so that the entire loop fits in one instruction-cache line.
15591 uint64_t LoopSize = 0;
15592 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I)
15593 for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) {
15594 LoopSize += TII->getInstSizeInBytes(*J);
15595 if (LoopSize > 32)
15596 break;
15597 }
15598
15599 if (LoopSize > 16 && LoopSize <= 32)
15600 return Align(32);
15601
15602 break;
15603 }
15604 }
15605
15606 return TargetLowering::getPrefLoopAlignment(ML);
15607}
15608
15609/// getConstraintType - Given a constraint, return the type of
15610/// constraint it is for this target.
15611PPCTargetLowering::ConstraintType
15612PPCTargetLowering::getConstraintType(StringRef Constraint) const {
15613 if (Constraint.size() == 1) {
15614 switch (Constraint[0]) {
15615 default: break;
15616 case 'b':
15617 case 'r':
15618 case 'f':
15619 case 'd':
15620 case 'v':
15621 case 'y':
15622 return C_RegisterClass;
15623 case 'Z':
15624 // FIXME: While Z does indicate a memory constraint, it specifically
15625 // indicates an r+r address (used in conjunction with the 'y' modifier
15626 // in the replacement string). Currently, we're forcing the base
15627 // register to be r0 in the asm printer (which is interpreted as zero)
15628 // and forming the complete address in the second register. This is
15629 // suboptimal.
15630 return C_Memory;
15631 }
15632 } else if (Constraint == "wc") { // individual CR bits.
15633 return C_RegisterClass;
15634 } else if (Constraint == "wa" || Constraint == "wd" ||
15635 Constraint == "wf" || Constraint == "ws" ||
15636 Constraint == "wi" || Constraint == "ww") {
15637 return C_RegisterClass; // VSX registers.
15638 }
15639 return TargetLowering::getConstraintType(Constraint);
15640}
15641
15642/// Examine constraint type and operand type and determine a weight value.
15643/// This object must already have been set up with the operand type
15644/// and the current alternative constraint selected.
15645TargetLowering::ConstraintWeight
15646PPCTargetLowering::getSingleConstraintMatchWeight(
15647 AsmOperandInfo &info, const char *constraint) const {
15648 ConstraintWeight weight = CW_Invalid;
15649 Value *CallOperandVal = info.CallOperandVal;
15650 // If we don't have a value, we can't do a match,
15651 // but allow it at the lowest weight.
15652 if (!CallOperandVal)
15653 return CW_Default;
15654 Type *type = CallOperandVal->getType();
15655
15656 // Look at the constraint type.
15657 if (StringRef(constraint) == "wc" && type->isIntegerTy(1))
15658 return CW_Register; // an individual CR bit.
15659 else if ((StringRef(constraint) == "wa" ||
15660 StringRef(constraint) == "wd" ||
15661 StringRef(constraint) == "wf") &&
15662 type->isVectorTy())
15663 return CW_Register;
15664 else if (StringRef(constraint) == "wi" && type->isIntegerTy(64))
15665 return CW_Register; // just hold 64-bit integers data.
15666 else if (StringRef(constraint) == "ws" && type->isDoubleTy())
15667 return CW_Register;
15668 else if (StringRef(constraint) == "ww" && type->isFloatTy())
15669 return CW_Register;
15670
15671 switch (*constraint) {
15672 default:
15673 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
15674 break;
15675 case 'b':
15676 if (type->isIntegerTy())
15677 weight = CW_Register;
15678 break;
15679 case 'f':
15680 if (type->isFloatTy())
15681 weight = CW_Register;
15682 break;
15683 case 'd':
15684 if (type->isDoubleTy())
15685 weight = CW_Register;
15686 break;
15687 case 'v':
15688 if (type->isVectorTy())
15689 weight = CW_Register;
15690 break;
15691 case 'y':
15692 weight = CW_Register;
15693 break;
15694 case 'Z':
15695 weight = CW_Memory;
15696 break;
15697 }
15698 return weight;
15699}
15700
15701std::pair<unsigned, const TargetRegisterClass *>
15702PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
15703 StringRef Constraint,
15704 MVT VT) const {
15705 if (Constraint.size() == 1) {
15706 // GCC RS6000 Constraint Letters
15707 switch (Constraint[0]) {
15708 case 'b': // R1-R31
15709 if (VT == MVT::i64 && Subtarget.isPPC64())
15710 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass);
15711 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass);
15712 case 'r': // R0-R31
15713 if (VT == MVT::i64 && Subtarget.isPPC64())
15714 return std::make_pair(0U, &PPC::G8RCRegClass);
15715 return std::make_pair(0U, &PPC::GPRCRegClass);
15716 // 'd' and 'f' constraints are both defined to be "the floating point
15717 // registers", where one is for 32-bit and the other for 64-bit. We don't
15718 // really care overly much here so just give them all the same reg classes.
15719 case 'd':
15720 case 'f':
15721 if (Subtarget.hasSPE()) {
15722 if (VT == MVT::f32 || VT == MVT::i32)
15723 return std::make_pair(0U, &PPC::GPRCRegClass);
15724 if (VT == MVT::f64 || VT == MVT::i64)
15725 return std::make_pair(0U, &PPC::SPERCRegClass);
15726 } else {
15727 if (VT == MVT::f32 || VT == MVT::i32)
15728 return std::make_pair(0U, &PPC::F4RCRegClass);
15729 if (VT == MVT::f64 || VT == MVT::i64)
15730 return std::make_pair(0U, &PPC::F8RCRegClass);
15731 }
15732 break;
15733 case 'v':
15734 if (Subtarget.hasAltivec())
15735 return std::make_pair(0U, &PPC::VRRCRegClass);
15736 break;
15737 case 'y': // crrc
15738 return std::make_pair(0U, &PPC::CRRCRegClass);
15739 }
15740 } else if (Constraint == "wc" && Subtarget.useCRBits()) {
15741 // An individual CR bit.
15742 return std::make_pair(0U, &PPC::CRBITRCRegClass);
15743 } else if ((Constraint == "wa" || Constraint == "wd" ||
15744 Constraint == "wf" || Constraint == "wi") &&
15745 Subtarget.hasVSX()) {
15746 // A VSX register for either a scalar (FP) or vector. There is no
15747 // support for single precision scalars on subtargets prior to Power8.
15748 if (VT.isVector())
15749 return std::make_pair(0U, &PPC::VSRCRegClass);
15750 if (VT == MVT::f32 && Subtarget.hasP8Vector())
15751 return std::make_pair(0U, &PPC::VSSRCRegClass);
15752 return std::make_pair(0U, &PPC::VSFRCRegClass);
15753 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) {
15754 if (VT == MVT::f32 && Subtarget.hasP8Vector())
15755 return std::make_pair(0U, &PPC::VSSRCRegClass);
15756 else
15757 return std::make_pair(0U, &PPC::VSFRCRegClass);
15758 } else if (Constraint == "lr") {
15759 if (VT == MVT::i64)
15760 return std::make_pair(0U, &PPC::LR8RCRegClass);
15761 else
15762 return std::make_pair(0U, &PPC::LRRCRegClass);
15763 }
15764
15765 // Handle special cases of physical registers that are not properly handled
15766 // by the base class.
15767 if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') {
15768 // If we name a VSX register, we can't defer to the base class because it
15769 // will not recognize the correct register (their names will be VSL{0-31}
15770 // and V{0-31} so they won't match). So we match them here.
15771 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') {
15772 int VSNum = atoi(Constraint.data() + 3);
15773 assert(VSNum >= 0 && VSNum <= 63 &&(static_cast <bool> (VSNum >= 0 && VSNum <=
63 && "Attempted to access a vsr out of range") ? void
(0) : __assert_fail ("VSNum >= 0 && VSNum <= 63 && \"Attempted to access a vsr out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15774, __extension__ __PRETTY_FUNCTION__))
15774 "Attempted to access a vsr out of range")(static_cast <bool> (VSNum >= 0 && VSNum <=
63 && "Attempted to access a vsr out of range") ? void
(0) : __assert_fail ("VSNum >= 0 && VSNum <= 63 && \"Attempted to access a vsr out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15774, __extension__ __PRETTY_FUNCTION__))
;
15775 if (VSNum < 32)
15776 return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass);
15777 return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass);
15778 }
15779
15780 // For float registers, we can't defer to the base class as it will match
15781 // the SPILLTOVSRRC class.
15782 if (Constraint.size() > 3 && Constraint[1] == 'f') {
15783 int RegNum = atoi(Constraint.data() + 2);
15784 if (RegNum > 31 || RegNum < 0)
15785 report_fatal_error("Invalid floating point register number");
15786 if (VT == MVT::f32 || VT == MVT::i32)
15787 return Subtarget.hasSPE()
15788 ? std::make_pair(PPC::R0 + RegNum, &PPC::GPRCRegClass)
15789 : std::make_pair(PPC::F0 + RegNum, &PPC::F4RCRegClass);
15790 if (VT == MVT::f64 || VT == MVT::i64)
15791 return Subtarget.hasSPE()
15792 ? std::make_pair(PPC::S0 + RegNum, &PPC::SPERCRegClass)
15793 : std::make_pair(PPC::F0 + RegNum, &PPC::F8RCRegClass);
15794 }
15795 }
15796
15797 std::pair<unsigned, const TargetRegisterClass *> R =
15798 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
15799
15800 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers
15801 // (which we call X[0-9]+). If a 64-bit value has been requested, and a
15802 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent
15803 // register.
15804 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use
15805 // the AsmName field from *RegisterInfo.td, then this would not be necessary.
15806 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() &&
15807 PPC::GPRCRegClass.contains(R.first))
15808 return std::make_pair(TRI->getMatchingSuperReg(R.first,
15809 PPC::sub_32, &PPC::G8RCRegClass),
15810 &PPC::G8RCRegClass);
15811
15812 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same.
15813 if (!R.second && StringRef("{cc}").equals_insensitive(Constraint)) {
15814 R.first = PPC::CR0;
15815 R.second = &PPC::CRRCRegClass;
15816 }
15817 // FIXME: This warning should ideally be emitted in the front end.
15818 const auto &TM = getTargetMachine();
15819 if (Subtarget.isAIXABI() && !TM.getAIXExtendedAltivecABI()) {
15820 if (((R.first >= PPC::V20 && R.first <= PPC::V31) ||
15821 (R.first >= PPC::VF20 && R.first <= PPC::VF31)) &&
15822 (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass))
15823 errs() << "warning: vector registers 20 to 32 are reserved in the "
15824 "default AIX AltiVec ABI and cannot be used\n";
15825 }
15826
15827 return R;
15828}
15829
15830/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
15831/// vector. If it is invalid, don't add anything to Ops.
15832void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
15833 std::string &Constraint,
15834 std::vector<SDValue>&Ops,
15835 SelectionDAG &DAG) const {
15836 SDValue Result;
15837
15838 // Only support length 1 constraints.
15839 if (Constraint.length() > 1) return;
15840
15841 char Letter = Constraint[0];
15842 switch (Letter) {
15843 default: break;
15844 case 'I':
15845 case 'J':
15846 case 'K':
15847 case 'L':
15848 case 'M':
15849 case 'N':
15850 case 'O':
15851 case 'P': {
15852 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op);
15853 if (!CST) return; // Must be an immediate to match.
15854 SDLoc dl(Op);
15855 int64_t Value = CST->getSExtValue();
15856 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
15857 // numbers are printed as such.
15858 switch (Letter) {
15859 default: llvm_unreachable("Unknown constraint letter!")::llvm::llvm_unreachable_internal("Unknown constraint letter!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 15859)
;
15860 case 'I': // "I" is a signed 16-bit constant.
15861 if (isInt<16>(Value))
15862 Result = DAG.getTargetConstant(Value, dl, TCVT);
15863 break;
15864 case 'J': // "J" is a constant with only the high-order 16 bits nonzero.
15865 if (isShiftedUInt<16, 16>(Value))
15866 Result = DAG.getTargetConstant(Value, dl, TCVT);
15867 break;
15868 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits.
15869 if (isShiftedInt<16, 16>(Value))
15870 Result = DAG.getTargetConstant(Value, dl, TCVT);
15871 break;
15872 case 'K': // "K" is a constant with only the low-order 16 bits nonzero.
15873 if (isUInt<16>(Value))
15874 Result = DAG.getTargetConstant(Value, dl, TCVT);
15875 break;
15876 case 'M': // "M" is a constant that is greater than 31.
15877 if (Value > 31)
15878 Result = DAG.getTargetConstant(Value, dl, TCVT);
15879 break;
15880 case 'N': // "N" is a positive constant that is an exact power of two.
15881 if (Value > 0 && isPowerOf2_64(Value))
15882 Result = DAG.getTargetConstant(Value, dl, TCVT);
15883 break;
15884 case 'O': // "O" is the constant zero.
15885 if (Value == 0)
15886 Result = DAG.getTargetConstant(Value, dl, TCVT);
15887 break;
15888 case 'P': // "P" is a constant whose negation is a signed 16-bit constant.
15889 if (isInt<16>(-Value))
15890 Result = DAG.getTargetConstant(Value, dl, TCVT);
15891 break;
15892 }
15893 break;
15894 }
15895 }
15896
15897 if (Result.getNode()) {
15898 Ops.push_back(Result);
15899 return;
15900 }
15901
15902 // Handle standard constraint letters.
15903 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
15904}
15905
15906// isLegalAddressingMode - Return true if the addressing mode represented
15907// by AM is legal for this target, for a load/store of the specified type.
15908bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
15909 const AddrMode &AM, Type *Ty,
15910 unsigned AS,
15911 Instruction *I) const {
15912 // Vector type r+i form is supported since power9 as DQ form. We don't check
15913 // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC,
15914 // imm form is preferred and the offset can be adjusted to use imm form later
15915 // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and
15916 // max offset to check legal addressing mode, we should be a little aggressive
15917 // to contain other offsets for that LSRUse.
15918 if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector())
15919 return false;
15920
15921 // PPC allows a sign-extended 16-bit immediate field.
15922 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
15923 return false;
15924
15925 // No global is ever allowed as a base.
15926 if (AM.BaseGV)
15927 return false;
15928
15929 // PPC only support r+r,
15930 switch (AM.Scale) {
15931 case 0: // "r+i" or just "i", depending on HasBaseReg.
15932 break;
15933 case 1:
15934 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
15935 return false;
15936 // Otherwise we have r+r or r+i.
15937 break;
15938 case 2:
15939 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
15940 return false;
15941 // Allow 2*r as r+r.
15942 break;
15943 default:
15944 // No other scales are supported.
15945 return false;
15946 }
15947
15948 return true;
15949}
15950
15951SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
15952 SelectionDAG &DAG) const {
15953 MachineFunction &MF = DAG.getMachineFunction();
15954 MachineFrameInfo &MFI = MF.getFrameInfo();
15955 MFI.setReturnAddressIsTaken(true);
15956
15957 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
15958 return SDValue();
15959
15960 SDLoc dl(Op);
15961 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
15962
15963 // Make sure the function does not optimize away the store of the RA to
15964 // the stack.
15965 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
15966 FuncInfo->setLRStoreRequired();
15967 bool isPPC64 = Subtarget.isPPC64();
15968 auto PtrVT = getPointerTy(MF.getDataLayout());
15969
15970 if (Depth > 0) {
15971 // The link register (return address) is saved in the caller's frame
15972 // not the callee's stack frame. So we must get the caller's frame
15973 // address and load the return address at the LR offset from there.
15974 SDValue FrameAddr =
15975 DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(),
15976 LowerFRAMEADDR(Op, DAG), MachinePointerInfo());
15977 SDValue Offset =
15978 DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl,
15979 isPPC64 ? MVT::i64 : MVT::i32);
15980 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
15981 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
15982 MachinePointerInfo());
15983 }
15984
15985 // Just load the return address off the stack.
15986 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
15987 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
15988 MachinePointerInfo());
15989}
15990
15991SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
15992 SelectionDAG &DAG) const {
15993 SDLoc dl(Op);
15994 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
15995
15996 MachineFunction &MF = DAG.getMachineFunction();
15997 MachineFrameInfo &MFI = MF.getFrameInfo();
15998 MFI.setFrameAddressIsTaken(true);
15999
16000 EVT PtrVT = getPointerTy(MF.getDataLayout());
16001 bool isPPC64 = PtrVT == MVT::i64;
16002
16003 // Naked functions never have a frame pointer, and so we use r1. For all
16004 // other functions, this decision must be delayed until during PEI.
16005 unsigned FrameReg;
16006 if (MF.getFunction().hasFnAttribute(Attribute::Naked))
16007 FrameReg = isPPC64 ? PPC::X1 : PPC::R1;
16008 else
16009 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP;
16010
16011 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg,
16012 PtrVT);
16013 while (Depth--)
16014 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(),
16015 FrameAddr, MachinePointerInfo());
16016 return FrameAddr;
16017}
16018
16019// FIXME? Maybe this could be a TableGen attribute on some registers and
16020// this table could be generated automatically from RegInfo.
16021Register PPCTargetLowering::getRegisterByName(const char* RegName, LLT VT,
16022 const MachineFunction &MF) const {
16023 bool isPPC64 = Subtarget.isPPC64();
16024
16025 bool is64Bit = isPPC64 && VT == LLT::scalar(64);
16026 if (!is64Bit && VT != LLT::scalar(32))
16027 report_fatal_error("Invalid register global variable type");
16028
16029 Register Reg = StringSwitch<Register>(RegName)
16030 .Case("r1", is64Bit ? PPC::X1 : PPC::R1)
16031 .Case("r2", isPPC64 ? Register() : PPC::R2)
16032 .Case("r13", (is64Bit ? PPC::X13 : PPC::R13))
16033 .Default(Register());
16034
16035 if (Reg)
16036 return Reg;
16037 report_fatal_error("Invalid register name global variable");
16038}
16039
16040bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
16041 // 32-bit SVR4 ABI access everything as got-indirect.
16042 if (Subtarget.is32BitELFABI())
16043 return true;
16044
16045 // AIX accesses everything indirectly through the TOC, which is similar to
16046 // the GOT.
16047 if (Subtarget.isAIXABI())
16048 return true;
16049
16050 CodeModel::Model CModel = getTargetMachine().getCodeModel();
16051 // If it is small or large code model, module locals are accessed
16052 // indirectly by loading their address from .toc/.got.
16053 if (CModel == CodeModel::Small || CModel == CodeModel::Large)
16054 return true;
16055
16056 // JumpTable and BlockAddress are accessed as got-indirect.
16057 if (isa<JumpTableSDNode>(GA) || isa<BlockAddressSDNode>(GA))
16058 return true;
16059
16060 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA))
16061 return Subtarget.isGVIndirectSymbol(G->getGlobal());
16062
16063 return false;
16064}
16065
16066bool
16067PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
16068 // The PowerPC target isn't yet aware of offsets.
16069 return false;
16070}
16071
16072bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
16073 const CallInst &I,
16074 MachineFunction &MF,
16075 unsigned Intrinsic) const {
16076 switch (Intrinsic) {
16077 case Intrinsic::ppc_atomicrmw_xchg_i128:
16078 case Intrinsic::ppc_atomicrmw_add_i128:
16079 case Intrinsic::ppc_atomicrmw_sub_i128:
16080 case Intrinsic::ppc_atomicrmw_nand_i128:
16081 case Intrinsic::ppc_atomicrmw_and_i128:
16082 case Intrinsic::ppc_atomicrmw_or_i128:
16083 case Intrinsic::ppc_atomicrmw_xor_i128:
16084 case Intrinsic::ppc_cmpxchg_i128:
16085 Info.opc = ISD::INTRINSIC_W_CHAIN;
16086 Info.memVT = MVT::i128;
16087 Info.ptrVal = I.getArgOperand(0);
16088 Info.offset = 0;
16089 Info.align = Align(16);
16090 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
16091 MachineMemOperand::MOVolatile;
16092 return true;
16093 case Intrinsic::ppc_altivec_lvx:
16094 case Intrinsic::ppc_altivec_lvxl:
16095 case Intrinsic::ppc_altivec_lvebx:
16096 case Intrinsic::ppc_altivec_lvehx:
16097 case Intrinsic::ppc_altivec_lvewx:
16098 case Intrinsic::ppc_vsx_lxvd2x:
16099 case Intrinsic::ppc_vsx_lxvw4x:
16100 case Intrinsic::ppc_vsx_lxvd2x_be:
16101 case Intrinsic::ppc_vsx_lxvw4x_be:
16102 case Intrinsic::ppc_vsx_lxvl:
16103 case Intrinsic::ppc_vsx_lxvll: {
16104 EVT VT;
16105 switch (Intrinsic) {
16106 case Intrinsic::ppc_altivec_lvebx:
16107 VT = MVT::i8;
16108 break;
16109 case Intrinsic::ppc_altivec_lvehx:
16110 VT = MVT::i16;
16111 break;
16112 case Intrinsic::ppc_altivec_lvewx:
16113 VT = MVT::i32;
16114 break;
16115 case Intrinsic::ppc_vsx_lxvd2x:
16116 case Intrinsic::ppc_vsx_lxvd2x_be:
16117 VT = MVT::v2f64;
16118 break;
16119 default:
16120 VT = MVT::v4i32;
16121 break;
16122 }
16123
16124 Info.opc = ISD::INTRINSIC_W_CHAIN;
16125 Info.memVT = VT;
16126 Info.ptrVal = I.getArgOperand(0);
16127 Info.offset = -VT.getStoreSize()+1;
16128 Info.size = 2*VT.getStoreSize()-1;
16129 Info.align = Align(1);
16130 Info.flags = MachineMemOperand::MOLoad;
16131 return true;
16132 }
16133 case Intrinsic::ppc_altivec_stvx:
16134 case Intrinsic::ppc_altivec_stvxl:
16135 case Intrinsic::ppc_altivec_stvebx:
16136 case Intrinsic::ppc_altivec_stvehx:
16137 case Intrinsic::ppc_altivec_stvewx:
16138 case Intrinsic::ppc_vsx_stxvd2x:
16139 case Intrinsic::ppc_vsx_stxvw4x:
16140 case Intrinsic::ppc_vsx_stxvd2x_be:
16141 case Intrinsic::ppc_vsx_stxvw4x_be:
16142 case Intrinsic::ppc_vsx_stxvl:
16143 case Intrinsic::ppc_vsx_stxvll: {
16144 EVT VT;
16145 switch (Intrinsic) {
16146 case Intrinsic::ppc_altivec_stvebx:
16147 VT = MVT::i8;
16148 break;
16149 case Intrinsic::ppc_altivec_stvehx:
16150 VT = MVT::i16;
16151 break;
16152 case Intrinsic::ppc_altivec_stvewx:
16153 VT = MVT::i32;
16154 break;
16155 case Intrinsic::ppc_vsx_stxvd2x:
16156 case Intrinsic::ppc_vsx_stxvd2x_be:
16157 VT = MVT::v2f64;
16158 break;
16159 default:
16160 VT = MVT::v4i32;
16161 break;
16162 }
16163
16164 Info.opc = ISD::INTRINSIC_VOID;
16165 Info.memVT = VT;
16166 Info.ptrVal = I.getArgOperand(1);
16167 Info.offset = -VT.getStoreSize()+1;
16168 Info.size = 2*VT.getStoreSize()-1;
16169 Info.align = Align(1);
16170 Info.flags = MachineMemOperand::MOStore;
16171 return true;
16172 }
16173 default:
16174 break;
16175 }
16176
16177 return false;
16178}
16179
16180/// It returns EVT::Other if the type should be determined using generic
16181/// target-independent logic.
16182EVT PPCTargetLowering::getOptimalMemOpType(
16183 const MemOp &Op, const AttributeList &FuncAttributes) const {
16184 if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
16185 // We should use Altivec/VSX loads and stores when available. For unaligned
16186 // addresses, unaligned VSX loads are only fast starting with the P8.
16187 if (Subtarget.hasAltivec() && Op.size() >= 16 &&
16188 (Op.isAligned(Align(16)) ||
16189 ((Op.isMemset() && Subtarget.hasVSX()) || Subtarget.hasP8Vector())))
16190 return MVT::v4i32;
16191 }
16192
16193 if (Subtarget.isPPC64()) {
16194 return MVT::i64;
16195 }
16196
16197 return MVT::i32;
16198}
16199
16200/// Returns true if it is beneficial to convert a load of a constant
16201/// to just the constant itself.
16202bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
16203 Type *Ty) const {
16204 assert(Ty->isIntegerTy())(static_cast <bool> (Ty->isIntegerTy()) ? void (0) :
__assert_fail ("Ty->isIntegerTy()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 16204, __extension__ __PRETTY_FUNCTION__))
;
16205
16206 unsigned BitSize = Ty->getPrimitiveSizeInBits();
16207 return !(BitSize == 0 || BitSize > 64);
16208}
16209
16210bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
16211 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
16212 return false;
16213 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
16214 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
16215 return NumBits1 == 64 && NumBits2 == 32;
16216}
16217
16218bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
16219 if (!VT1.isInteger() || !VT2.isInteger())
16220 return false;
16221 unsigned NumBits1 = VT1.getSizeInBits();
16222 unsigned NumBits2 = VT2.getSizeInBits();
16223 return NumBits1 == 64 && NumBits2 == 32;
16224}
16225
16226bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
16227 // Generally speaking, zexts are not free, but they are free when they can be
16228 // folded with other operations.
16229 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) {
16230 EVT MemVT = LD->getMemoryVT();
16231 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 ||
16232 (Subtarget.isPPC64() && MemVT == MVT::i32)) &&
16233 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
16234 LD->getExtensionType() == ISD::ZEXTLOAD))
16235 return true;
16236 }
16237
16238 // FIXME: Add other cases...
16239 // - 32-bit shifts with a zext to i64
16240 // - zext after ctlz, bswap, etc.
16241 // - zext after and by a constant mask
16242
16243 return TargetLowering::isZExtFree(Val, VT2);
16244}
16245
16246bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const {
16247 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&(static_cast <bool> (DestVT.isFloatingPoint() &&
SrcVT.isFloatingPoint() && "invalid fpext types") ? void
(0) : __assert_fail ("DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && \"invalid fpext types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 16248, __extension__ __PRETTY_FUNCTION__))
16248 "invalid fpext types")(static_cast <bool> (DestVT.isFloatingPoint() &&
SrcVT.isFloatingPoint() && "invalid fpext types") ? void
(0) : __assert_fail ("DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && \"invalid fpext types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 16248, __extension__ __PRETTY_FUNCTION__))
;
16249 // Extending to float128 is not free.
16250 if (DestVT == MVT::f128)
16251 return false;
16252 return true;
16253}
16254
16255bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
16256 return isInt<16>(Imm) || isUInt<16>(Imm);
16257}
16258
16259bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
16260 return isInt<16>(Imm) || isUInt<16>(Imm);
16261}
16262
16263bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
16264 MachineMemOperand::Flags,
16265 bool *Fast) const {
16266 if (DisablePPCUnaligned)
16267 return false;
16268
16269 // PowerPC supports unaligned memory access for simple non-vector types.
16270 // Although accessing unaligned addresses is not as efficient as accessing
16271 // aligned addresses, it is generally more efficient than manual expansion,
16272 // and generally only traps for software emulation when crossing page
16273 // boundaries.
16274
16275 if (!VT.isSimple())
16276 return false;
16277
16278 if (VT.isFloatingPoint() && !VT.isVector() &&
16279 !Subtarget.allowsUnalignedFPAccess())
16280 return false;
16281
16282 if (VT.getSimpleVT().isVector()) {
16283 if (Subtarget.hasVSX()) {
16284 if (VT != MVT::v2f64 && VT != MVT::v2i64 &&
16285 VT != MVT::v4f32 && VT != MVT::v4i32)
16286 return false;
16287 } else {
16288 return false;
16289 }
16290 }
16291
16292 if (VT == MVT::ppcf128)
16293 return false;
16294
16295 if (Fast)
16296 *Fast = true;
16297
16298 return true;
16299}
16300
16301bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
16302 SDValue C) const {
16303 // Check integral scalar types.
16304 if (!VT.isScalarInteger())
16305 return false;
16306 if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
16307 if (!ConstNode->getAPIntValue().isSignedIntN(64))
16308 return false;
16309 // This transformation will generate >= 2 operations. But the following
16310 // cases will generate <= 2 instructions during ISEL. So exclude them.
16311 // 1. If the constant multiplier fits 16 bits, it can be handled by one
16312 // HW instruction, ie. MULLI
16313 // 2. If the multiplier after shifted fits 16 bits, an extra shift
16314 // instruction is needed than case 1, ie. MULLI and RLDICR
16315 int64_t Imm = ConstNode->getSExtValue();
16316 unsigned Shift = countTrailingZeros<uint64_t>(Imm);
16317 Imm >>= Shift;
16318 if (isInt<16>(Imm))
16319 return false;
16320 uint64_t UImm = static_cast<uint64_t>(Imm);
16321 if (isPowerOf2_64(UImm + 1) || isPowerOf2_64(UImm - 1) ||
16322 isPowerOf2_64(1 - UImm) || isPowerOf2_64(-1 - UImm))
16323 return true;
16324 }
16325 return false;
16326}
16327
16328bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
16329 EVT VT) const {
16330 return isFMAFasterThanFMulAndFAdd(
16331 MF.getFunction(), VT.getTypeForEVT(MF.getFunction().getContext()));
16332}
16333
16334bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
16335 Type *Ty) const {
16336 switch (Ty->getScalarType()->getTypeID()) {
16337 case Type::FloatTyID:
16338 case Type::DoubleTyID:
16339 return true;
16340 case Type::FP128TyID:
16341 return Subtarget.hasP9Vector();
16342 default:
16343 return false;
16344 }
16345}
16346
16347// FIXME: add more patterns which are not profitable to hoist.
16348bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const {
16349 if (!I->hasOneUse())
16350 return true;
16351
16352 Instruction *User = I->user_back();
16353 assert(User && "A single use instruction with no uses.")(static_cast <bool> (User && "A single use instruction with no uses."
) ? void (0) : __assert_fail ("User && \"A single use instruction with no uses.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 16353, __extension__ __PRETTY_FUNCTION__))
;
16354
16355 switch (I->getOpcode()) {
16356 case Instruction::FMul: {
16357 // Don't break FMA, PowerPC prefers FMA.
16358 if (User->getOpcode() != Instruction::FSub &&
16359 User->getOpcode() != Instruction::FAdd)
16360 return true;
16361
16362 const TargetOptions &Options = getTargetMachine().Options;
16363 const Function *F = I->getFunction();
16364 const DataLayout &DL = F->getParent()->getDataLayout();
16365 Type *Ty = User->getOperand(0)->getType();
16366
16367 return !(
16368 isFMAFasterThanFMulAndFAdd(*F, Ty) &&
16369 isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) &&
16370 (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath));
16371 }
16372 case Instruction::Load: {
16373 // Don't break "store (load float*)" pattern, this pattern will be combined
16374 // to "store (load int32)" in later InstCombine pass. See function
16375 // combineLoadToOperationType. On PowerPC, loading a float point takes more
16376 // cycles than loading a 32 bit integer.
16377 LoadInst *LI = cast<LoadInst>(I);
16378 // For the loads that combineLoadToOperationType does nothing, like
16379 // ordered load, it should be profitable to hoist them.
16380 // For swifterror load, it can only be used for pointer to pointer type, so
16381 // later type check should get rid of this case.
16382 if (!LI->isUnordered())
16383 return true;
16384
16385 if (User->getOpcode() != Instruction::Store)
16386 return true;
16387
16388 if (I->getType()->getTypeID() != Type::FloatTyID)
16389 return true;
16390
16391 return false;
16392 }
16393 default:
16394 return true;
16395 }
16396 return true;
16397}
16398
16399const MCPhysReg *
16400PPCTargetLowering::getScratchRegisters(CallingConv::ID) const {
16401 // LR is a callee-save register, but we must treat it as clobbered by any call
16402 // site. Hence we include LR in the scratch registers, which are in turn added
16403 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies
16404 // to CTR, which is used by any indirect call.
16405 static const MCPhysReg ScratchRegs[] = {
16406 PPC::X12, PPC::LR8, PPC::CTR8, 0
16407 };
16408
16409 return ScratchRegs;
16410}
16411
16412Register PPCTargetLowering::getExceptionPointerRegister(
16413 const Constant *PersonalityFn) const {
16414 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3;
16415}
16416
16417Register PPCTargetLowering::getExceptionSelectorRegister(
16418 const Constant *PersonalityFn) const {
16419 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4;
16420}
16421
16422bool
16423PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
16424 EVT VT , unsigned DefinedValues) const {
16425 if (VT == MVT::v2i64)
16426 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
16427
16428 if (Subtarget.hasVSX())
16429 return true;
16430
16431 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
16432}
16433
16434Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const {
16435 if (DisableILPPref || Subtarget.enableMachineScheduler())
16436 return TargetLowering::getSchedulingPreference(N);
16437
16438 return Sched::ILP;
16439}
16440
16441// Create a fast isel object.
16442FastISel *
16443PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo,
16444 const TargetLibraryInfo *LibInfo) const {
16445 return PPC::createFastISel(FuncInfo, LibInfo);
16446}
16447
16448// 'Inverted' means the FMA opcode after negating one multiplicand.
16449// For example, (fma -a b c) = (fnmsub a b c)
16450static unsigned invertFMAOpcode(unsigned Opc) {
16451 switch (Opc) {
16452 default:
16453 llvm_unreachable("Invalid FMA opcode for PowerPC!")::llvm::llvm_unreachable_internal("Invalid FMA opcode for PowerPC!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 16453)
;
16454 case ISD::FMA:
16455 return PPCISD::FNMSUB;
16456 case PPCISD::FNMSUB:
16457 return ISD::FMA;
16458 }
16459}
16460
16461SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
16462 bool LegalOps, bool OptForSize,
16463 NegatibleCost &Cost,
16464 unsigned Depth) const {
16465 if (Depth > SelectionDAG::MaxRecursionDepth)
16466 return SDValue();
16467
16468 unsigned Opc = Op.getOpcode();
16469 EVT VT = Op.getValueType();
16470 SDNodeFlags Flags = Op.getNode()->getFlags();
16471
16472 switch (Opc) {
16473 case PPCISD::FNMSUB:
16474 if (!Op.hasOneUse() || !isTypeLegal(VT))
16475 break;
16476
16477 const TargetOptions &Options = getTargetMachine().Options;
16478 SDValue N0 = Op.getOperand(0);
16479 SDValue N1 = Op.getOperand(1);
16480 SDValue N2 = Op.getOperand(2);
16481 SDLoc Loc(Op);
16482
16483 NegatibleCost N2Cost = NegatibleCost::Expensive;
16484 SDValue NegN2 =
16485 getNegatedExpression(N2, DAG, LegalOps, OptForSize, N2Cost, Depth + 1);
16486
16487 if (!NegN2)
16488 return SDValue();
16489
16490 // (fneg (fnmsub a b c)) => (fnmsub (fneg a) b (fneg c))
16491 // (fneg (fnmsub a b c)) => (fnmsub a (fneg b) (fneg c))
16492 // These transformations may change sign of zeroes. For example,
16493 // -(-ab-(-c))=-0 while -(-(ab-c))=+0 when a=b=c=1.
16494 if (Flags.hasNoSignedZeros() || Options.NoSignedZerosFPMath) {
16495 // Try and choose the cheaper one to negate.
16496 NegatibleCost N0Cost = NegatibleCost::Expensive;
16497 SDValue NegN0 = getNegatedExpression(N0, DAG, LegalOps, OptForSize,
16498 N0Cost, Depth + 1);
16499
16500 NegatibleCost N1Cost = NegatibleCost::Expensive;
16501 SDValue NegN1 = getNegatedExpression(N1, DAG, LegalOps, OptForSize,
16502 N1Cost, Depth + 1);
16503
16504 if (NegN0 && N0Cost <= N1Cost) {
16505 Cost = std::min(N0Cost, N2Cost);
16506 return DAG.getNode(Opc, Loc, VT, NegN0, N1, NegN2, Flags);
16507 } else if (NegN1) {
16508 Cost = std::min(N1Cost, N2Cost);
16509 return DAG.getNode(Opc, Loc, VT, N0, NegN1, NegN2, Flags);
16510 }
16511 }
16512
16513 // (fneg (fnmsub a b c)) => (fma a b (fneg c))
16514 if (isOperationLegal(ISD::FMA, VT)) {
16515 Cost = N2Cost;
16516 return DAG.getNode(ISD::FMA, Loc, VT, N0, N1, NegN2, Flags);
16517 }
16518
16519 break;
16520 }
16521
16522 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps, OptForSize,
16523 Cost, Depth);
16524}
16525
16526// Override to enable LOAD_STACK_GUARD lowering on Linux.
16527bool PPCTargetLowering::useLoadStackGuardNode() const {
16528 if (!Subtarget.isTargetLinux())
16529 return TargetLowering::useLoadStackGuardNode();
16530 return true;
16531}
16532
16533// Override to disable global variable loading on Linux and insert AIX canary
16534// word declaration.
16535void PPCTargetLowering::insertSSPDeclarations(Module &M) const {
16536 if (Subtarget.isAIXABI()) {
16537 M.getOrInsertGlobal(AIXSSPCanaryWordName,
16538 Type::getInt8PtrTy(M.getContext()));
16539 return;
16540 }
16541 if (!Subtarget.isTargetLinux())
16542 return TargetLowering::insertSSPDeclarations(M);
16543}
16544
16545Value *PPCTargetLowering::getSDagStackGuard(const Module &M) const {
16546 if (Subtarget.isAIXABI())
16547 return M.getGlobalVariable(AIXSSPCanaryWordName);
16548 return TargetLowering::getSDagStackGuard(M);
16549}
16550
16551bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
16552 bool ForCodeSize) const {
16553 if (!VT.isSimple() || !Subtarget.hasVSX())
16554 return false;
16555
16556 switch(VT.getSimpleVT().SimpleTy) {
16557 default:
16558 // For FP types that are currently not supported by PPC backend, return
16559 // false. Examples: f16, f80.
16560 return false;
16561 case MVT::f32:
16562 case MVT::f64:
16563 if (Subtarget.hasPrefixInstrs()) {
16564 // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP.
16565 return true;
16566 }
16567 LLVM_FALLTHROUGH[[gnu::fallthrough]];
16568 case MVT::ppcf128:
16569 return Imm.isPosZero();
16570 }
16571}
16572
16573// For vector shift operation op, fold
16574// (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y)
16575static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N,
16576 SelectionDAG &DAG) {
16577 SDValue N0 = N->getOperand(0);
16578 SDValue N1 = N->getOperand(1);
16579 EVT VT = N0.getValueType();
16580 unsigned OpSizeInBits = VT.getScalarSizeInBits();
16581 unsigned Opcode = N->getOpcode();
16582 unsigned TargetOpcode;
16583
16584 switch (Opcode) {
16585 default:
16586 llvm_unreachable("Unexpected shift operation")::llvm::llvm_unreachable_internal("Unexpected shift operation"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 16586)
;
16587 case ISD::SHL:
16588 TargetOpcode = PPCISD::SHL;
16589 break;
16590 case ISD::SRL:
16591 TargetOpcode = PPCISD::SRL;
16592 break;
16593 case ISD::SRA:
16594 TargetOpcode = PPCISD::SRA;
16595 break;
16596 }
16597
16598 if (VT.isVector() && TLI.isOperationLegal(Opcode, VT) &&
16599 N1->getOpcode() == ISD::AND)
16600 if (ConstantSDNode *Mask = isConstOrConstSplat(N1->getOperand(1)))
16601 if (Mask->getZExtValue() == OpSizeInBits - 1)
16602 return DAG.getNode(TargetOpcode, SDLoc(N), VT, N0, N1->getOperand(0));
16603
16604 return SDValue();
16605}
16606
16607SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const {
16608 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG))
16609 return Value;
16610
16611 SDValue N0 = N->getOperand(0);
16612 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N->getOperand(1));
16613 if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() ||
16614 N0.getOpcode() != ISD::SIGN_EXTEND ||
16615 N0.getOperand(0).getValueType() != MVT::i32 || CN1 == nullptr ||
16616 N->getValueType(0) != MVT::i64)
16617 return SDValue();
16618
16619 // We can't save an operation here if the value is already extended, and
16620 // the existing shift is easier to combine.
16621 SDValue ExtsSrc = N0.getOperand(0);
16622 if (ExtsSrc.getOpcode() == ISD::TRUNCATE &&
16623 ExtsSrc.getOperand(0).getOpcode() == ISD::AssertSext)
16624 return SDValue();
16625
16626 SDLoc DL(N0);
16627 SDValue ShiftBy = SDValue(CN1, 0);
16628 // We want the shift amount to be i32 on the extswli, but the shift could
16629 // have an i64.
16630 if (ShiftBy.getValueType() == MVT::i64)
16631 ShiftBy = DCI.DAG.getConstant(CN1->getZExtValue(), DL, MVT::i32);
16632
16633 return DCI.DAG.getNode(PPCISD::EXTSWSLI, DL, MVT::i64, N0->getOperand(0),
16634 ShiftBy);
16635}
16636
16637SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const {
16638 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG))
16639 return Value;
16640
16641 return SDValue();
16642}
16643
16644SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const {
16645 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG))
16646 return Value;
16647
16648 return SDValue();
16649}
16650
16651// Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1))
16652// Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0))
16653// When C is zero, the equation (addi Z, -C) can be simplified to Z
16654// Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types
16655static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
16656 const PPCSubtarget &Subtarget) {
16657 if (!Subtarget.isPPC64())
16658 return SDValue();
16659
16660 SDValue LHS = N->getOperand(0);
16661 SDValue RHS = N->getOperand(1);
16662
16663 auto isZextOfCompareWithConstant = [](SDValue Op) {
16664 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() ||
16665 Op.getValueType() != MVT::i64)
16666 return false;
16667
16668 SDValue Cmp = Op.getOperand(0);
16669 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() ||
16670 Cmp.getOperand(0).getValueType() != MVT::i64)
16671 return false;
16672
16673 if (auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1))) {
16674 int64_t NegConstant = 0 - Constant->getSExtValue();
16675 // Due to the limitations of the addi instruction,
16676 // -C is required to be [-32768, 32767].
16677 return isInt<16>(NegConstant);
16678 }
16679
16680 return false;
16681 };
16682
16683 bool LHSHasPattern = isZextOfCompareWithConstant(LHS);
16684 bool RHSHasPattern = isZextOfCompareWithConstant(RHS);
16685
16686 // If there is a pattern, canonicalize a zext operand to the RHS.
16687 if (LHSHasPattern && !RHSHasPattern)
16688 std::swap(LHS, RHS);
16689 else if (!LHSHasPattern && !RHSHasPattern)
16690 return SDValue();
16691
16692 SDLoc DL(N);
16693 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue);
16694 SDValue Cmp = RHS.getOperand(0);
16695 SDValue Z = Cmp.getOperand(0);
16696 auto *Constant = cast<ConstantSDNode>(Cmp.getOperand(1));
16697 int64_t NegConstant = 0 - Constant->getSExtValue();
16698
16699 switch(cast<CondCodeSDNode>(Cmp.getOperand(2))->get()) {
16700 default: break;
16701 case ISD::SETNE: {
16702 // when C == 0
16703 // --> addze X, (addic Z, -1).carry
16704 // /
16705 // add X, (zext(setne Z, C))--
16706 // \ when -32768 <= -C <= 32767 && C != 0
16707 // --> addze X, (addic (addi Z, -C), -1).carry
16708 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z,
16709 DAG.getConstant(NegConstant, DL, MVT::i64));
16710 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
16711 SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue),
16712 AddOrZ, DAG.getConstant(-1ULL, DL, MVT::i64));
16713 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64),
16714 SDValue(Addc.getNode(), 1));
16715 }
16716 case ISD::SETEQ: {
16717 // when C == 0
16718 // --> addze X, (subfic Z, 0).carry
16719 // /
16720 // add X, (zext(sete Z, C))--
16721 // \ when -32768 <= -C <= 32767 && C != 0
16722 // --> addze X, (subfic (addi Z, -C), 0).carry
16723 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z,
16724 DAG.getConstant(NegConstant, DL, MVT::i64));
16725 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
16726 SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue),
16727 DAG.getConstant(0, DL, MVT::i64), AddOrZ);
16728 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64),
16729 SDValue(Subc.getNode(), 1));
16730 }
16731 }
16732
16733 return SDValue();
16734}
16735
16736// Transform
16737// (add C1, (MAT_PCREL_ADDR GlobalAddr+C2)) to
16738// (MAT_PCREL_ADDR GlobalAddr+(C1+C2))
16739// In this case both C1 and C2 must be known constants.
16740// C1+C2 must fit into a 34 bit signed integer.
16741static SDValue combineADDToMAT_PCREL_ADDR(SDNode *N, SelectionDAG &DAG,
16742 const PPCSubtarget &Subtarget) {
16743 if (!Subtarget.isUsingPCRelativeCalls())
16744 return SDValue();
16745
16746 // Check both Operand 0 and Operand 1 of the ADD node for the PCRel node.
16747 // If we find that node try to cast the Global Address and the Constant.
16748 SDValue LHS = N->getOperand(0);
16749 SDValue RHS = N->getOperand(1);
16750
16751 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
16752 std::swap(LHS, RHS);
16753
16754 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
16755 return SDValue();
16756
16757 // Operand zero of PPCISD::MAT_PCREL_ADDR is the GA node.
16758 GlobalAddressSDNode *GSDN = dyn_cast<GlobalAddressSDNode>(LHS.getOperand(0));
16759 ConstantSDNode* ConstNode = dyn_cast<ConstantSDNode>(RHS);
16760
16761 // Check that both casts succeeded.
16762 if (!GSDN || !ConstNode)
16763 return SDValue();
16764
16765 int64_t NewOffset = GSDN->getOffset() + ConstNode->getSExtValue();
16766 SDLoc DL(GSDN);
16767
16768 // The signed int offset needs to fit in 34 bits.
16769 if (!isInt<34>(NewOffset))
16770 return SDValue();
16771
16772 // The new global address is a copy of the old global address except
16773 // that it has the updated Offset.
16774 SDValue GA =
16775 DAG.getTargetGlobalAddress(GSDN->getGlobal(), DL, GSDN->getValueType(0),
16776 NewOffset, GSDN->getTargetFlags());
16777 SDValue MatPCRel =
16778 DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, GSDN->getValueType(0), GA);
16779 return MatPCRel;
16780}
16781
16782SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const {
16783 if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget))
16784 return Value;
16785
16786 if (auto Value = combineADDToMAT_PCREL_ADDR(N, DCI.DAG, Subtarget))
16787 return Value;
16788
16789 return SDValue();
16790}
16791
16792// Detect TRUNCATE operations on bitcasts of float128 values.
16793// What we are looking for here is the situtation where we extract a subset
16794// of bits from a 128 bit float.
16795// This can be of two forms:
16796// 1) BITCAST of f128 feeding TRUNCATE
16797// 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE
16798// The reason this is required is because we do not have a legal i128 type
16799// and so we want to prevent having to store the f128 and then reload part
16800// of it.
16801SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N,
16802 DAGCombinerInfo &DCI) const {
16803 // If we are using CRBits then try that first.
16804 if (Subtarget.useCRBits()) {
16805 // Check if CRBits did anything and return that if it did.
16806 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI))
16807 return CRTruncValue;
16808 }
16809
16810 SDLoc dl(N);
16811 SDValue Op0 = N->getOperand(0);
16812
16813 // fold (truncate (abs (sub (zext a), (zext b)))) -> (vabsd a, b)
16814 if (Subtarget.hasP9Altivec() && Op0.getOpcode() == ISD::ABS) {
16815 EVT VT = N->getValueType(0);
16816 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8)
16817 return SDValue();
16818 SDValue Sub = Op0.getOperand(0);
16819 if (Sub.getOpcode() == ISD::SUB) {
16820 SDValue SubOp0 = Sub.getOperand(0);
16821 SDValue SubOp1 = Sub.getOperand(1);
16822 if ((SubOp0.getOpcode() == ISD::ZERO_EXTEND) &&
16823 (SubOp1.getOpcode() == ISD::ZERO_EXTEND)) {
16824 return DCI.DAG.getNode(PPCISD::VABSD, dl, VT, SubOp0.getOperand(0),
16825 SubOp1.getOperand(0),
16826 DCI.DAG.getTargetConstant(0, dl, MVT::i32));
16827 }
16828 }
16829 }
16830
16831 // Looking for a truncate of i128 to i64.
16832 if (Op0.getValueType() != MVT::i128 || N->getValueType(0) != MVT::i64)
16833 return SDValue();
16834
16835 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0;
16836
16837 // SRL feeding TRUNCATE.
16838 if (Op0.getOpcode() == ISD::SRL) {
16839 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Op0.getOperand(1));
16840 // The right shift has to be by 64 bits.
16841 if (!ConstNode || ConstNode->getZExtValue() != 64)
16842 return SDValue();
16843
16844 // Switch the element number to extract.
16845 EltToExtract = EltToExtract ? 0 : 1;
16846 // Update Op0 past the SRL.
16847 Op0 = Op0.getOperand(0);
16848 }
16849
16850 // BITCAST feeding a TRUNCATE possibly via SRL.
16851 if (Op0.getOpcode() == ISD::BITCAST &&
16852 Op0.getValueType() == MVT::i128 &&
16853 Op0.getOperand(0).getValueType() == MVT::f128) {
16854 SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0));
16855 return DCI.DAG.getNode(
16856 ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Bitcast,
16857 DCI.DAG.getTargetConstant(EltToExtract, dl, MVT::i32));
16858 }
16859 return SDValue();
16860}
16861
16862SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const {
16863 SelectionDAG &DAG = DCI.DAG;
16864
16865 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N->getOperand(1));
16866 if (!ConstOpOrElement)
16867 return SDValue();
16868
16869 // An imul is usually smaller than the alternative sequence for legal type.
16870 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
16871 isOperationLegal(ISD::MUL, N->getValueType(0)))
16872 return SDValue();
16873
16874 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool {
16875 switch (this->Subtarget.getCPUDirective()) {
16876 default:
16877 // TODO: enhance the condition for subtarget before pwr8
16878 return false;
16879 case PPC::DIR_PWR8:
16880 // type mul add shl
16881 // scalar 4 1 1
16882 // vector 7 2 2
16883 return true;
16884 case PPC::DIR_PWR9:
16885 case PPC::DIR_PWR10:
16886 case PPC::DIR_PWR_FUTURE:
16887 // type mul add shl
16888 // scalar 5 2 2
16889 // vector 7 2 2
16890
16891 // The cycle RATIO of related operations are showed as a table above.
16892 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both
16893 // scalar and vector type. For 2 instrs patterns, add/sub + shl
16894 // are 4, it is always profitable; but for 3 instrs patterns
16895 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6.
16896 // So we should only do it for vector type.
16897 return IsAddOne && IsNeg ? VT.isVector() : true;
16898 }
16899 };
16900
16901 EVT VT = N->getValueType(0);
16902 SDLoc DL(N);
16903
16904 const APInt &MulAmt = ConstOpOrElement->getAPIntValue();
16905 bool IsNeg = MulAmt.isNegative();
16906 APInt MulAmtAbs = MulAmt.abs();
16907
16908 if ((MulAmtAbs - 1).isPowerOf2()) {
16909 // (mul x, 2^N + 1) => (add (shl x, N), x)
16910 // (mul x, -(2^N + 1)) => -(add (shl x, N), x)
16911
16912 if (!IsProfitable(IsNeg, true, VT))
16913 return SDValue();
16914
16915 SDValue Op0 = N->getOperand(0);
16916 SDValue Op1 =
16917 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
16918 DAG.getConstant((MulAmtAbs - 1).logBase2(), DL, VT));
16919 SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1);
16920
16921 if (!IsNeg)
16922 return Res;
16923
16924 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res);
16925 } else if ((MulAmtAbs + 1).isPowerOf2()) {
16926 // (mul x, 2^N - 1) => (sub (shl x, N), x)
16927 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
16928
16929 if (!IsProfitable(IsNeg, false, VT))
16930 return SDValue();
16931
16932 SDValue Op0 = N->getOperand(0);
16933 SDValue Op1 =
16934 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
16935 DAG.getConstant((MulAmtAbs + 1).logBase2(), DL, VT));
16936
16937 if (!IsNeg)
16938 return DAG.getNode(ISD::SUB, DL, VT, Op1, Op0);
16939 else
16940 return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1);
16941
16942 } else {
16943 return SDValue();
16944 }
16945}
16946
16947// Combine fma-like op (like fnmsub) with fnegs to appropriate op. Do this
16948// in combiner since we need to check SD flags and other subtarget features.
16949SDValue PPCTargetLowering::combineFMALike(SDNode *N,
16950 DAGCombinerInfo &DCI) const {
16951 SDValue N0 = N->getOperand(0);
16952 SDValue N1 = N->getOperand(1);
16953 SDValue N2 = N->getOperand(2);
16954 SDNodeFlags Flags = N->getFlags();
16955 EVT VT = N->getValueType(0);
16956 SelectionDAG &DAG = DCI.DAG;
16957 const TargetOptions &Options = getTargetMachine().Options;
16958 unsigned Opc = N->getOpcode();
16959 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
16960 bool LegalOps = !DCI.isBeforeLegalizeOps();
16961 SDLoc Loc(N);
16962
16963 if (!isOperationLegal(ISD::FMA, VT))
16964 return SDValue();
16965
16966 // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
16967 // since (fnmsub a b c)=-0 while c-ab=+0.
16968 if (!Flags.hasNoSignedZeros() && !Options.NoSignedZerosFPMath)
16969 return SDValue();
16970
16971 // (fma (fneg a) b c) => (fnmsub a b c)
16972 // (fnmsub (fneg a) b c) => (fma a b c)
16973 if (SDValue NegN0 = getCheaperNegatedExpression(N0, DAG, LegalOps, CodeSize))
16974 return DAG.getNode(invertFMAOpcode(Opc), Loc, VT, NegN0, N1, N2, Flags);
16975
16976 // (fma a (fneg b) c) => (fnmsub a b c)
16977 // (fnmsub a (fneg b) c) => (fma a b c)
16978 if (SDValue NegN1 = getCheaperNegatedExpression(N1, DAG, LegalOps, CodeSize))
16979 return DAG.getNode(invertFMAOpcode(Opc), Loc, VT, N0, NegN1, N2, Flags);
16980
16981 return SDValue();
16982}
16983
16984bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
16985 // Only duplicate to increase tail-calls for the 64bit SysV ABIs.
16986 if (!Subtarget.is64BitELFABI())
16987 return false;
16988
16989 // If not a tail call then no need to proceed.
16990 if (!CI->isTailCall())
16991 return false;
16992
16993 // If sibling calls have been disabled and tail-calls aren't guaranteed
16994 // there is no reason to duplicate.
16995 auto &TM = getTargetMachine();
16996 if (!TM.Options.GuaranteedTailCallOpt && DisableSCO)
16997 return false;
16998
16999 // Can't tail call a function called indirectly, or if it has variadic args.
17000 const Function *Callee = CI->getCalledFunction();
17001 if (!Callee || Callee->isVarArg())
17002 return false;
17003
17004 // Make sure the callee and caller calling conventions are eligible for tco.
17005 const Function *Caller = CI->getParent()->getParent();
17006 if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(),
17007 CI->getCallingConv()))
17008 return false;
17009
17010 // If the function is local then we have a good chance at tail-calling it
17011 return getTargetMachine().shouldAssumeDSOLocal(*Caller->getParent(), Callee);
17012}
17013
17014bool PPCTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
17015 if (!Subtarget.hasVSX())
17016 return false;
17017 if (Subtarget.hasP9Vector() && VT == MVT::f128)
17018 return true;
17019 return VT == MVT::f32 || VT == MVT::f64 ||
17020 VT == MVT::v4f32 || VT == MVT::v2f64;
17021}
17022
17023bool PPCTargetLowering::
17024isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
17025 const Value *Mask = AndI.getOperand(1);
17026 // If the mask is suitable for andi. or andis. we should sink the and.
17027 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Mask)) {
17028 // Can't handle constants wider than 64-bits.
17029 if (CI->getBitWidth() > 64)
17030 return false;
17031 int64_t ConstVal = CI->getZExtValue();
17032 return isUInt<16>(ConstVal) ||
17033 (isUInt<16>(ConstVal >> 16) && !(ConstVal & 0xFFFF));
17034 }
17035
17036 // For non-constant masks, we can always use the record-form and.
17037 return true;
17038}
17039
17040// Transform (abs (sub (zext a), (zext b))) to (vabsd a b 0)
17041// Transform (abs (sub (zext a), (zext_invec b))) to (vabsd a b 0)
17042// Transform (abs (sub (zext_invec a), (zext_invec b))) to (vabsd a b 0)
17043// Transform (abs (sub (zext_invec a), (zext b))) to (vabsd a b 0)
17044// Transform (abs (sub a, b) to (vabsd a b 1)) if a & b of type v4i32
17045SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const {
17046 assert((N->getOpcode() == ISD::ABS) && "Need ABS node here")(static_cast <bool> ((N->getOpcode() == ISD::ABS) &&
"Need ABS node here") ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ABS) && \"Need ABS node here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17046, __extension__ __PRETTY_FUNCTION__))
;
17047 assert(Subtarget.hasP9Altivec() &&(static_cast <bool> (Subtarget.hasP9Altivec() &&
"Only combine this when P9 altivec supported!") ? void (0) :
__assert_fail ("Subtarget.hasP9Altivec() && \"Only combine this when P9 altivec supported!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17048, __extension__ __PRETTY_FUNCTION__))
17048 "Only combine this when P9 altivec supported!")(static_cast <bool> (Subtarget.hasP9Altivec() &&
"Only combine this when P9 altivec supported!") ? void (0) :
__assert_fail ("Subtarget.hasP9Altivec() && \"Only combine this when P9 altivec supported!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17048, __extension__ __PRETTY_FUNCTION__))
;
17049 EVT VT = N->getValueType(0);
17050 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8)
17051 return SDValue();
17052
17053 SelectionDAG &DAG = DCI.DAG;
17054 SDLoc dl(N);
17055 if (N->getOperand(0).getOpcode() == ISD::SUB) {
17056 // Even for signed integers, if it's known to be positive (as signed
17057 // integer) due to zero-extended inputs.
17058 unsigned SubOpcd0 = N->getOperand(0)->getOperand(0).getOpcode();
17059 unsigned SubOpcd1 = N->getOperand(0)->getOperand(1).getOpcode();
17060 if ((SubOpcd0 == ISD::ZERO_EXTEND ||
17061 SubOpcd0 == ISD::ZERO_EXTEND_VECTOR_INREG) &&
17062 (SubOpcd1 == ISD::ZERO_EXTEND ||
17063 SubOpcd1 == ISD::ZERO_EXTEND_VECTOR_INREG)) {
17064 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(),
17065 N->getOperand(0)->getOperand(0),
17066 N->getOperand(0)->getOperand(1),
17067 DAG.getTargetConstant(0, dl, MVT::i32));
17068 }
17069
17070 // For type v4i32, it can be optimized with xvnegsp + vabsduw
17071 if (N->getOperand(0).getValueType() == MVT::v4i32 &&
17072 N->getOperand(0).hasOneUse()) {
17073 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(),
17074 N->getOperand(0)->getOperand(0),
17075 N->getOperand(0)->getOperand(1),
17076 DAG.getTargetConstant(1, dl, MVT::i32));
17077 }
17078 }
17079
17080 return SDValue();
17081}
17082
17083// For type v4i32/v8ii16/v16i8, transform
17084// from (vselect (setcc a, b, setugt), (sub a, b), (sub b, a)) to (vabsd a, b)
17085// from (vselect (setcc a, b, setuge), (sub a, b), (sub b, a)) to (vabsd a, b)
17086// from (vselect (setcc a, b, setult), (sub b, a), (sub a, b)) to (vabsd a, b)
17087// from (vselect (setcc a, b, setule), (sub b, a), (sub a, b)) to (vabsd a, b)
17088SDValue PPCTargetLowering::combineVSelect(SDNode *N,
17089 DAGCombinerInfo &DCI) const {
17090 assert((N->getOpcode() == ISD::VSELECT) && "Need VSELECT node here")(static_cast <bool> ((N->getOpcode() == ISD::VSELECT
) && "Need VSELECT node here") ? void (0) : __assert_fail
("(N->getOpcode() == ISD::VSELECT) && \"Need VSELECT node here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17090, __extension__ __PRETTY_FUNCTION__))
;
17091 assert(Subtarget.hasP9Altivec() &&(static_cast <bool> (Subtarget.hasP9Altivec() &&
"Only combine this when P9 altivec supported!") ? void (0) :
__assert_fail ("Subtarget.hasP9Altivec() && \"Only combine this when P9 altivec supported!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17092, __extension__ __PRETTY_FUNCTION__))
17092 "Only combine this when P9 altivec supported!")(static_cast <bool> (Subtarget.hasP9Altivec() &&
"Only combine this when P9 altivec supported!") ? void (0) :
__assert_fail ("Subtarget.hasP9Altivec() && \"Only combine this when P9 altivec supported!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17092, __extension__ __PRETTY_FUNCTION__))
;
17093
17094 SelectionDAG &DAG = DCI.DAG;
17095 SDLoc dl(N);
17096 SDValue Cond = N->getOperand(0);
17097 SDValue TrueOpnd = N->getOperand(1);
17098 SDValue FalseOpnd = N->getOperand(2);
17099 EVT VT = N->getOperand(1).getValueType();
17100
17101 if (Cond.getOpcode() != ISD::SETCC || TrueOpnd.getOpcode() != ISD::SUB ||
17102 FalseOpnd.getOpcode() != ISD::SUB)
17103 return SDValue();
17104
17105 // ABSD only available for type v4i32/v8i16/v16i8
17106 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8)
17107 return SDValue();
17108
17109 // At least to save one more dependent computation
17110 if (!(Cond.hasOneUse() || TrueOpnd.hasOneUse() || FalseOpnd.hasOneUse()))
17111 return SDValue();
17112
17113 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
17114
17115 // Can only handle unsigned comparison here
17116 switch (CC) {
17117 default:
17118 return SDValue();
17119 case ISD::SETUGT:
17120 case ISD::SETUGE:
17121 break;
17122 case ISD::SETULT:
17123 case ISD::SETULE:
17124 std::swap(TrueOpnd, FalseOpnd);
17125 break;
17126 }
17127
17128 SDValue CmpOpnd1 = Cond.getOperand(0);
17129 SDValue CmpOpnd2 = Cond.getOperand(1);
17130
17131 // SETCC CmpOpnd1 CmpOpnd2 cond
17132 // TrueOpnd = CmpOpnd1 - CmpOpnd2
17133 // FalseOpnd = CmpOpnd2 - CmpOpnd1
17134 if (TrueOpnd.getOperand(0) == CmpOpnd1 &&
17135 TrueOpnd.getOperand(1) == CmpOpnd2 &&
17136 FalseOpnd.getOperand(0) == CmpOpnd2 &&
17137 FalseOpnd.getOperand(1) == CmpOpnd1) {
17138 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(1).getValueType(),
17139 CmpOpnd1, CmpOpnd2,
17140 DAG.getTargetConstant(0, dl, MVT::i32));
17141 }
17142
17143 return SDValue();
17144}
17145
17146/// getAddrModeForFlags - Based on the set of address flags, select the most
17147/// optimal instruction format to match by.
17148PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const {
17149 // This is not a node we should be handling here.
17150 if (Flags == PPC::MOF_None)
17151 return PPC::AM_None;
17152 // Unaligned D-Forms are tried first, followed by the aligned D-Forms.
17153 for (auto FlagSet : AddrModesMap.at(PPC::AM_DForm))
17154 if ((Flags & FlagSet) == FlagSet)
17155 return PPC::AM_DForm;
17156 for (auto FlagSet : AddrModesMap.at(PPC::AM_DSForm))
17157 if ((Flags & FlagSet) == FlagSet)
17158 return PPC::AM_DSForm;
17159 for (auto FlagSet : AddrModesMap.at(PPC::AM_DQForm))
17160 if ((Flags & FlagSet) == FlagSet)
17161 return PPC::AM_DQForm;
17162 // If no other forms are selected, return an X-Form as it is the most
17163 // general addressing mode.
17164 return PPC::AM_XForm;
17165}
17166
17167/// Set alignment flags based on whether or not the Frame Index is aligned.
17168/// Utilized when computing flags for address computation when selecting
17169/// load and store instructions.
17170static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet,
17171 SelectionDAG &DAG) {
17172 bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR));
17173 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(IsAdd ? N.getOperand(0) : N);
17174 if (!FI)
17175 return;
17176 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
17177 unsigned FrameIndexAlign = MFI.getObjectAlign(FI->getIndex()).value();
17178 // If this is (add $FI, $S16Imm), the alignment flags are already set
17179 // based on the immediate. We just need to clear the alignment flags
17180 // if the FI alignment is weaker.
17181 if ((FrameIndexAlign % 4) != 0)
17182 FlagSet &= ~PPC::MOF_RPlusSImm16Mult4;
17183 if ((FrameIndexAlign % 16) != 0)
17184 FlagSet &= ~PPC::MOF_RPlusSImm16Mult16;
17185 // If the address is a plain FrameIndex, set alignment flags based on
17186 // FI alignment.
17187 if (!IsAdd) {
17188 if ((FrameIndexAlign % 4) == 0)
17189 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
17190 if ((FrameIndexAlign % 16) == 0)
17191 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
17192 }
17193}
17194
17195/// Given a node, compute flags that are used for address computation when
17196/// selecting load and store instructions. The flags computed are stored in
17197/// FlagSet. This function takes into account whether the node is a constant,
17198/// an ADD, OR, or a constant, and computes the address flags accordingly.
17199static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet,
17200 SelectionDAG &DAG) {
17201 // Set the alignment flags for the node depending on if the node is
17202 // 4-byte or 16-byte aligned.
17203 auto SetAlignFlagsForImm = [&](uint64_t Imm) {
17204 if ((Imm & 0x3) == 0)
17205 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
17206 if ((Imm & 0xf) == 0)
17207 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
17208 };
17209
17210 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) {
20
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
35
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
36
Assuming 'CN' is null
17211 // All 32-bit constants can be computed as LIS + Disp.
17212 const APInt &ConstImm = CN->getAPIntValue();
17213 if (ConstImm.isSignedIntN(32)) { // Flag to handle 32-bit constants.
17214 FlagSet |= PPC::MOF_AddrIsSImm32;
17215 SetAlignFlagsForImm(ConstImm.getZExtValue());
17216 setAlignFlagsForFI(N, FlagSet, DAG);
17217 }
17218 if (ConstImm.isSignedIntN(34)) // Flag to handle 34-bit constants.
17219 FlagSet |= PPC::MOF_RPlusSImm34;
17220 else // Let constant materialization handle large constants.
17221 FlagSet |= PPC::MOF_NotAddNorCst;
17222 } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) {
37
Calling 'SDValue::getOpcode'
17223 // This address can be represented as an addition of:
17224 // - Register + Imm16 (possibly a multiple of 4/16)
17225 // - Register + Imm34
17226 // - Register + PPCISD::Lo
17227 // - Register + Register
17228 // In any case, we won't have to match this as Base + Zero.
17229 SDValue RHS = N.getOperand(1);
17230 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(RHS)) {
17231 const APInt &ConstImm = CN->getAPIntValue();
17232 if (ConstImm.isSignedIntN(16)) {
17233 FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates.
17234 SetAlignFlagsForImm(ConstImm.getZExtValue());
17235 setAlignFlagsForFI(N, FlagSet, DAG);
17236 }
17237 if (ConstImm.isSignedIntN(34))
17238 FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates.
17239 else
17240 FlagSet |= PPC::MOF_RPlusR; // Register.
17241 } else if (RHS.getOpcode() == PPCISD::Lo &&
17242 !cast<ConstantSDNode>(RHS.getOperand(1))->getZExtValue())
17243 FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo.
17244 else
17245 FlagSet |= PPC::MOF_RPlusR;
17246 } else { // The address computation is not a constant or an addition.
17247 setAlignFlagsForFI(N, FlagSet, DAG);
17248 FlagSet |= PPC::MOF_NotAddNorCst;
17249 }
17250}
17251
17252/// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute
17253/// the address flags of the load/store instruction that is to be matched.
17254unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
17255 SelectionDAG &DAG) const {
17256 unsigned FlagSet = PPC::MOF_None;
17257
17258 // Compute subtarget flags.
17259 if (!Subtarget.hasP9Vector())
2
Assuming the condition is false
3
Taking false branch
17260 FlagSet |= PPC::MOF_SubtargetBeforeP9;
17261 else {
17262 FlagSet |= PPC::MOF_SubtargetP9;
17263 if (Subtarget.hasPrefixInstrs())
4
Assuming the condition is false
5
Taking false branch
17264 FlagSet |= PPC::MOF_SubtargetP10;
17265 }
17266 if (Subtarget.hasSPE())
6
Assuming the condition is false
7
Taking false branch
17267 FlagSet |= PPC::MOF_SubtargetSPE;
17268
17269 // Mark this as something we don't want to handle here if it is atomic
17270 // or pre-increment instruction.
17271 if (const LSBaseSDNode *LSB
8.1
'LSB' is null
8.1
'LSB' is null
8.1
'LSB' is null
= dyn_cast<LSBaseSDNode>(Parent))
8
Assuming 'Parent' is not a 'LSBaseSDNode'
9
Taking false branch
17272 if (LSB->isIndexed())
17273 return PPC::MOF_None;
17274
17275 // Compute in-memory type flags. This is based on if there are scalars,
17276 // floats or vectors.
17277 const MemSDNode *MN = dyn_cast<MemSDNode>(Parent);
10
Assuming 'Parent' is a 'MemSDNode'
17278 assert(MN && "Parent should be a MemSDNode!")(static_cast <bool> (MN && "Parent should be a MemSDNode!"
) ? void (0) : __assert_fail ("MN && \"Parent should be a MemSDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17278, __extension__ __PRETTY_FUNCTION__))
;
11
'?' condition is true
17279 EVT MemVT = MN->getMemoryVT();
17280 unsigned Size = MemVT.getSizeInBits();
17281 if (MemVT.isScalarInteger()) {
12
Assuming the condition is true
17282 assert(Size <= 64 && "Not expecting scalar integers larger than 8 bytes!")(static_cast <bool> (Size <= 64 && "Not expecting scalar integers larger than 8 bytes!"
) ? void (0) : __assert_fail ("Size <= 64 && \"Not expecting scalar integers larger than 8 bytes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17282, __extension__ __PRETTY_FUNCTION__))
;
13
Taking true branch
14
Assuming 'Size' is <= 64
15
'?' condition is true
17283 if (Size < 32)
16
Assuming 'Size' is < 32
17
Taking true branch
17284 FlagSet |= PPC::MOF_SubWordInt;
17285 else if (Size == 32)
17286 FlagSet |= PPC::MOF_WordInt;
17287 else
17288 FlagSet |= PPC::MOF_DoubleWordInt;
17289 } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors.
17290 if (Size == 128)
17291 FlagSet |= PPC::MOF_Vector;
17292 else if (Size == 256)
17293 FlagSet |= PPC::MOF_Vector256;
17294 else
17295 llvm_unreachable("Not expecting illegal vectors!")::llvm::llvm_unreachable_internal("Not expecting illegal vectors!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17295)
;
17296 } else { // Floating point type: can be scalar, f128 or vector types.
17297 if (Size == 32 || Size == 64)
17298 FlagSet |= PPC::MOF_ScalarFloat;
17299 else if (MemVT == MVT::f128 || MemVT.isVector())
17300 FlagSet |= PPC::MOF_Vector;
17301 else
17302 llvm_unreachable("Not expecting illegal scalar floats!")::llvm::llvm_unreachable_internal("Not expecting illegal scalar floats!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17302)
;
17303 }
17304
17305 // Compute flags for address computation.
17306 computeFlagsForAddressComputation(N, FlagSet, DAG);
18
The value of 'N' is assigned to 'N.Node'
19
Calling 'computeFlagsForAddressComputation'
17307
17308 // Compute type extension flags.
17309 if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Parent)) {
17310 switch (LN->getExtensionType()) {
17311 case ISD::SEXTLOAD:
17312 FlagSet |= PPC::MOF_SExt;
17313 break;
17314 case ISD::EXTLOAD:
17315 case ISD::ZEXTLOAD:
17316 FlagSet |= PPC::MOF_ZExt;
17317 break;
17318 case ISD::NON_EXTLOAD:
17319 FlagSet |= PPC::MOF_NoExt;
17320 break;
17321 }
17322 } else
17323 FlagSet |= PPC::MOF_NoExt;
17324
17325 // For integers, no extension is the same as zero extension.
17326 // We set the extension mode to zero extension so we don't have
17327 // to add separate entries in AddrModesMap for loads and stores.
17328 if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) {
17329 FlagSet |= PPC::MOF_ZExt;
17330 FlagSet &= ~PPC::MOF_NoExt;
17331 }
17332
17333 // If we don't have prefixed instructions, 34-bit constants should be
17334 // treated as PPC::MOF_NotAddNorCst so they can match D-Forms.
17335 bool IsNonP1034BitConst =
17336 ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) &
17337 FlagSet) == PPC::MOF_RPlusSImm34;
17338 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR &&
17339 IsNonP1034BitConst)
17340 FlagSet |= PPC::MOF_NotAddNorCst;
17341
17342 return FlagSet;
17343}
17344
17345/// SelectForceXFormMode - Given the specified address, force it to be
17346/// represented as an indexed [r+r] operation (an XForm instruction).
17347PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp,
17348 SDValue &Base,
17349 SelectionDAG &DAG) const {
17350
17351 PPC::AddrMode Mode = PPC::AM_XForm;
17352 int16_t ForceXFormImm = 0;
17353 if (provablyDisjointOr(DAG, N) &&
17354 !isIntS16Immediate(N.getOperand(1), ForceXFormImm)) {
17355 Disp = N.getOperand(0);
17356 Base = N.getOperand(1);
17357 return Mode;
17358 }
17359
17360 // If the address is the result of an add, we will utilize the fact that the
17361 // address calculation includes an implicit add. However, we can reduce
17362 // register pressure if we do not materialize a constant just for use as the
17363 // index register. We only get rid of the add if it is not an add of a
17364 // value and a 16-bit signed constant and both have a single use.
17365 if (N.getOpcode() == ISD::ADD &&
17366 (!isIntS16Immediate(N.getOperand(1), ForceXFormImm) ||
17367 !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) {
17368 Disp = N.getOperand(0);
17369 Base = N.getOperand(1);
17370 return Mode;
17371 }
17372
17373 // Otherwise, use R0 as the base register.
17374 Disp = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
17375 N.getValueType());
17376 Base = N;
17377
17378 return Mode;
17379}
17380
17381// If we happen to match to an aligned D-Form, check if the Frame Index is
17382// adequately aligned. If it is not, reset the mode to match to X-Form.
17383static void setXFormForUnalignedFI(SDValue N, unsigned Flags,
17384 PPC::AddrMode &Mode) {
17385 if (!isa<FrameIndexSDNode>(N))
17386 return;
17387 if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) ||
17388 (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16)))
17389 Mode = PPC::AM_XForm;
17390}
17391
17392/// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode),
17393/// compute the address flags of the node, get the optimal address mode based
17394/// on the flags, and set the Base and Disp based on the address mode.
17395PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent,
17396 SDValue N, SDValue &Disp,
17397 SDValue &Base,
17398 SelectionDAG &DAG,
17399 MaybeAlign Align) const {
17400 SDLoc DL(Parent);
17401
17402 // Compute the address flags.
17403 unsigned Flags = computeMOFlags(Parent, N, DAG);
1
Calling 'PPCTargetLowering::computeMOFlags'
17404
17405 // Get the optimal address mode based on the Flags.
17406 PPC::AddrMode Mode = getAddrModeForFlags(Flags);
17407
17408 // If the address mode is DS-Form or DQ-Form, check if the FI is aligned.
17409 // Select an X-Form load if it is not.
17410 setXFormForUnalignedFI(N, Flags, Mode);
17411
17412 // Set Base and Disp accordingly depending on the address mode.
17413 switch (Mode) {
17414 case PPC::AM_DForm:
17415 case PPC::AM_DSForm:
17416 case PPC::AM_DQForm: {
17417 // This is a register plus a 16-bit immediate. The base will be the
17418 // register and the displacement will be the immediate unless it
17419 // isn't sufficiently aligned.
17420 if (Flags & PPC::MOF_RPlusSImm16) {
17421 SDValue Op0 = N.getOperand(0);
17422 SDValue Op1 = N.getOperand(1);
17423 int16_t Imm = cast<ConstantSDNode>(Op1)->getAPIntValue().getZExtValue();
17424 if (!Align || isAligned(*Align, Imm)) {
17425 Disp = DAG.getTargetConstant(Imm, DL, N.getValueType());
17426 Base = Op0;
17427 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0)) {
17428 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
17429 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
17430 }
17431 break;
17432 }
17433 }
17434 // This is a register plus the @lo relocation. The base is the register
17435 // and the displacement is the global address.
17436 else if (Flags & PPC::MOF_RPlusLo) {
17437 Disp = N.getOperand(1).getOperand(0); // The global address.
17438 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17441, __extension__ __PRETTY_FUNCTION__))
17439 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17441, __extension__ __PRETTY_FUNCTION__))
17440 Disp.getOpcode() == ISD::TargetConstantPool ||(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17441, __extension__ __PRETTY_FUNCTION__))
17441 Disp.getOpcode() == ISD::TargetJumpTable)(static_cast <bool> (Disp.getOpcode() == ISD::TargetGlobalAddress
|| Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode
() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable
) ? void (0) : __assert_fail ("Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17441, __extension__ __PRETTY_FUNCTION__))
;
17442 Base = N.getOperand(0);
17443 break;
17444 }
17445 // This is a constant address at most 32 bits. The base will be
17446 // zero or load-immediate-shifted and the displacement will be
17447 // the low 16 bits of the address.
17448 else if (Flags & PPC::MOF_AddrIsSImm32) {
17449 auto *CN = cast<ConstantSDNode>(N);
17450 EVT CNType = CN->getValueType(0);
17451 uint64_t CNImm = CN->getZExtValue();
17452 // If this address fits entirely in a 16-bit sext immediate field, codegen
17453 // this as "d, 0".
17454 int16_t Imm;
17455 if (isIntS16Immediate(CN, Imm) && (!Align || isAligned(*Align, Imm))) {
17456 Disp = DAG.getTargetConstant(Imm, DL, CNType);
17457 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
17458 CNType);
17459 break;
17460 }
17461 // Handle 32-bit sext immediate with LIS + Addr mode.
17462 if ((CNType == MVT::i32 || isInt<32>(CNImm)) &&
17463 (!Align || isAligned(*Align, CNImm))) {
17464 int32_t Addr = (int32_t)CNImm;
17465 // Otherwise, break this down into LIS + Disp.
17466 Disp = DAG.getTargetConstant((int16_t)Addr, DL, MVT::i32);
17467 Base =
17468 DAG.getTargetConstant((Addr - (int16_t)Addr) >> 16, DL, MVT::i32);
17469 uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8;
17470 Base = SDValue(DAG.getMachineNode(LIS, DL, CNType, Base), 0);
17471 break;
17472 }
17473 }
17474 // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable.
17475 Disp = DAG.getTargetConstant(0, DL, getPointerTy(DAG.getDataLayout()));
17476 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) {
17477 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
17478 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
17479 } else
17480 Base = N;
17481 break;
17482 }
17483 case PPC::AM_None:
17484 break;
17485 default: { // By default, X-Form is always available to be selected.
17486 // When a frame index is not aligned, we also match by XForm.
17487 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
17488 Base = FI ? N : N.getOperand(1);
17489 Disp = FI ? DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
17490 N.getValueType())
17491 : N.getOperand(0);
17492 break;
17493 }
17494 }
17495 return Mode;
17496}
17497
17498CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC,
17499 bool Return,
17500 bool IsVarArg) const {
17501 switch (CC) {
17502 case CallingConv::Cold:
17503 return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF_FIS);
17504 default:
17505 return CC_PPC64_ELF_FIS;
17506 }
17507}
17508
17509TargetLowering::AtomicExpansionKind
17510PPCTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
17511 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
17512 if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && Size == 128)
17513 return AtomicExpansionKind::MaskedIntrinsic;
17514 return TargetLowering::shouldExpandAtomicRMWInIR(AI);
17515}
17516
17517TargetLowering::AtomicExpansionKind
17518PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
17519 unsigned Size = AI->getPointerOperand()
17520 ->getType()
17521 ->getPointerElementType()
17522 ->getPrimitiveSizeInBits();
17523 if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && Size == 128)
17524 return AtomicExpansionKind::MaskedIntrinsic;
17525 return TargetLowering::shouldExpandAtomicCmpXchgInIR(AI);
17526}
17527
17528static Intrinsic::ID
17529getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) {
17530 switch (BinOp) {
17531 default:
17532 llvm_unreachable("Unexpected AtomicRMW BinOp")::llvm::llvm_unreachable_internal("Unexpected AtomicRMW BinOp"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17532)
;
17533 case AtomicRMWInst::Xchg:
17534 return Intrinsic::ppc_atomicrmw_xchg_i128;
17535 case AtomicRMWInst::Add:
17536 return Intrinsic::ppc_atomicrmw_add_i128;
17537 case AtomicRMWInst::Sub:
17538 return Intrinsic::ppc_atomicrmw_sub_i128;
17539 case AtomicRMWInst::And:
17540 return Intrinsic::ppc_atomicrmw_and_i128;
17541 case AtomicRMWInst::Or:
17542 return Intrinsic::ppc_atomicrmw_or_i128;
17543 case AtomicRMWInst::Xor:
17544 return Intrinsic::ppc_atomicrmw_xor_i128;
17545 case AtomicRMWInst::Nand:
17546 return Intrinsic::ppc_atomicrmw_nand_i128;
17547 }
17548}
17549
17550Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic(
17551 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
17552 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
17553 assert(EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() &&(static_cast <bool> (EnableQuadwordAtomics && Subtarget
.hasQuadwordAtomics() && "Only support quadword now")
? void (0) : __assert_fail ("EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && \"Only support quadword now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17554, __extension__ __PRETTY_FUNCTION__))
17554 "Only support quadword now")(static_cast <bool> (EnableQuadwordAtomics && Subtarget
.hasQuadwordAtomics() && "Only support quadword now")
? void (0) : __assert_fail ("EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && \"Only support quadword now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17554, __extension__ __PRETTY_FUNCTION__))
;
17555 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
17556 Type *ValTy = cast<PointerType>(AlignedAddr->getType())->getElementType();
17557 assert(ValTy->getPrimitiveSizeInBits() == 128)(static_cast <bool> (ValTy->getPrimitiveSizeInBits()
== 128) ? void (0) : __assert_fail ("ValTy->getPrimitiveSizeInBits() == 128"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17557, __extension__ __PRETTY_FUNCTION__))
;
17558 Function *RMW = Intrinsic::getDeclaration(
17559 M, getIntrinsicForAtomicRMWBinOp128(AI->getOperation()));
17560 Type *Int64Ty = Type::getInt64Ty(M->getContext());
17561 Value *IncrLo = Builder.CreateTrunc(Incr, Int64Ty, "incr_lo");
17562 Value *IncrHi =
17563 Builder.CreateTrunc(Builder.CreateLShr(Incr, 64), Int64Ty, "incr_hi");
17564 Value *Addr =
17565 Builder.CreateBitCast(AlignedAddr, Type::getInt8PtrTy(M->getContext()));
17566 Value *LoHi = Builder.CreateCall(RMW, {Addr, IncrLo, IncrHi});
17567 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
17568 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
17569 Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
17570 Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
17571 return Builder.CreateOr(
17572 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64");
17573}
17574
17575Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
17576 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
17577 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
17578 assert(EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() &&(static_cast <bool> (EnableQuadwordAtomics && Subtarget
.hasQuadwordAtomics() && "Only support quadword now")
? void (0) : __assert_fail ("EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && \"Only support quadword now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17579, __extension__ __PRETTY_FUNCTION__))
17579 "Only support quadword now")(static_cast <bool> (EnableQuadwordAtomics && Subtarget
.hasQuadwordAtomics() && "Only support quadword now")
? void (0) : __assert_fail ("EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && \"Only support quadword now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17579, __extension__ __PRETTY_FUNCTION__))
;
17580 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
17581 Type *ValTy = cast<PointerType>(AlignedAddr->getType())->getElementType();
17582 assert(ValTy->getPrimitiveSizeInBits() == 128)(static_cast <bool> (ValTy->getPrimitiveSizeInBits()
== 128) ? void (0) : __assert_fail ("ValTy->getPrimitiveSizeInBits() == 128"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/PowerPC/PPCISelLowering.cpp"
, 17582, __extension__ __PRETTY_FUNCTION__))
;
17583 Function *IntCmpXchg =
17584 Intrinsic::getDeclaration(M, Intrinsic::ppc_cmpxchg_i128);
17585 Type *Int64Ty = Type::getInt64Ty(M->getContext());
17586 Value *CmpLo = Builder.CreateTrunc(CmpVal, Int64Ty, "cmp_lo");
17587 Value *CmpHi =
17588 Builder.CreateTrunc(Builder.CreateLShr(CmpVal, 64), Int64Ty, "cmp_hi");
17589 Value *NewLo = Builder.CreateTrunc(NewVal, Int64Ty, "new_lo");
17590 Value *NewHi =
17591 Builder.CreateTrunc(Builder.CreateLShr(NewVal, 64), Int64Ty, "new_hi");
17592 Value *Addr =
17593 Builder.CreateBitCast(AlignedAddr, Type::getInt8PtrTy(M->getContext()));
17594 emitLeadingFence(Builder, CI, Ord);
17595 Value *LoHi =
17596 Builder.CreateCall(IntCmpXchg, {Addr, CmpLo, CmpHi, NewLo, NewHi});
17597 emitTrailingFence(Builder, CI, Ord);
17598 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
17599 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
17600 Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
17601 Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
17602 return Builder.CreateOr(
17603 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64");
17604}

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h

1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
10// and dyn_cast_or_null<X>() templates.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_CASTING_H
15#define LLVM_SUPPORT_CASTING_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/type_traits.h"
19#include <cassert>
20#include <memory>
21#include <type_traits>
22
23namespace llvm {
24
25//===----------------------------------------------------------------------===//
26// isa<x> Support Templates
27//===----------------------------------------------------------------------===//
28
29// Define a template that can be specialized by smart pointers to reflect the
30// fact that they are automatically dereferenced, and are not involved with the
31// template selection process... the default implementation is a noop.
32//
33template<typename From> struct simplify_type {
34 using SimpleType = From; // The real type this represents...
35
36 // An accessor to get the real value...
37 static SimpleType &getSimplifiedValue(From &Val) { return Val; }
38};
39
40template<typename From> struct simplify_type<const From> {
41 using NonConstSimpleType = typename simplify_type<From>::SimpleType;
42 using SimpleType =
43 typename add_const_past_pointer<NonConstSimpleType>::type;
44 using RetType =
45 typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
46
47 static RetType getSimplifiedValue(const From& Val) {
48 return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
49 }
50};
51
52// The core of the implementation of isa<X> is here; To and From should be
53// the names of classes. This template can be specialized to customize the
54// implementation of isa<> without rewriting it from scratch.
55template <typename To, typename From, typename Enabler = void>
56struct isa_impl {
57 static inline bool doit(const From &Val) {
58 return To::classof(&Val);
59 }
60};
61
62/// Always allow upcasts, and perform no dynamic check for them.
63template <typename To, typename From>
64struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
65 static inline bool doit(const From &) { return true; }
66};
67
68template <typename To, typename From> struct isa_impl_cl {
69 static inline bool doit(const From &Val) {
70 return isa_impl<To, From>::doit(Val);
71 }
72};
73
74template <typename To, typename From> struct isa_impl_cl<To, const From> {
75 static inline bool doit(const From &Val) {
76 return isa_impl<To, From>::doit(Val);
77 }
78};
79
80template <typename To, typename From>
81struct isa_impl_cl<To, const std::unique_ptr<From>> {
82 static inline bool doit(const std::unique_ptr<From> &Val) {
83 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 83, __extension__ __PRETTY_FUNCTION__))
;
84 return isa_impl_cl<To, From>::doit(*Val);
85 }
86};
87
88template <typename To, typename From> struct isa_impl_cl<To, From*> {
89 static inline bool doit(const From *Val) {
90 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 90, __extension__ __PRETTY_FUNCTION__))
;
91 return isa_impl<To, From>::doit(*Val);
92 }
93};
94
95template <typename To, typename From> struct isa_impl_cl<To, From*const> {
96 static inline bool doit(const From *Val) {
97 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 97, __extension__ __PRETTY_FUNCTION__))
;
98 return isa_impl<To, From>::doit(*Val);
99 }
100};
101
102template <typename To, typename From> struct isa_impl_cl<To, const From*> {
103 static inline bool doit(const From *Val) {
104 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 104, __extension__ __PRETTY_FUNCTION__))
;
105 return isa_impl<To, From>::doit(*Val);
106 }
107};
108
109template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
110 static inline bool doit(const From *Val) {
111 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 111, __extension__ __PRETTY_FUNCTION__))
;
112 return isa_impl<To, From>::doit(*Val);
113 }
114};
115
116template<typename To, typename From, typename SimpleFrom>
117struct isa_impl_wrap {
118 // When From != SimplifiedType, we can simplify the type some more by using
119 // the simplify_type template.
120 static bool doit(const From &Val) {
121 return isa_impl_wrap<To, SimpleFrom,
122 typename simplify_type<SimpleFrom>::SimpleType>::doit(
123 simplify_type<const From>::getSimplifiedValue(Val));
124 }
125};
126
127template<typename To, typename FromTy>
128struct isa_impl_wrap<To, FromTy, FromTy> {
129 // When From == SimpleType, we are as simple as we are going to get.
130 static bool doit(const FromTy &Val) {
131 return isa_impl_cl<To,FromTy>::doit(Val);
132 }
133};
134
135// isa<X> - Return true if the parameter to the template is an instance of one
136// of the template type arguments. Used like this:
137//
138// if (isa<Type>(myVal)) { ... }
139// if (isa<Type0, Type1, Type2>(myVal)) { ... }
140//
141template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
142 return isa_impl_wrap<X, const Y,
143 typename simplify_type<const Y>::SimpleType>::doit(Val);
144}
145
146template <typename First, typename Second, typename... Rest, typename Y>
147LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
148 return isa<First>(Val) || isa<Second, Rest...>(Val);
149}
150
151// isa_and_nonnull<X> - Functionally identical to isa, except that a null value
152// is accepted.
153//
154template <typename... X, class Y>
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa_and_nonnull(const Y &Val) {
156 if (!Val)
157 return false;
158 return isa<X...>(Val);
159}
160
161//===----------------------------------------------------------------------===//
162// cast<x> Support Templates
163//===----------------------------------------------------------------------===//
164
165template<class To, class From> struct cast_retty;
166
167// Calculate what type the 'cast' function should return, based on a requested
168// type of To and a source type of From.
169template<class To, class From> struct cast_retty_impl {
170 using ret_type = To &; // Normal case, return Ty&
171};
172template<class To, class From> struct cast_retty_impl<To, const From> {
173 using ret_type = const To &; // Normal case, return Ty&
174};
175
176template<class To, class From> struct cast_retty_impl<To, From*> {
177 using ret_type = To *; // Pointer arg case, return Ty*
178};
179
180template<class To, class From> struct cast_retty_impl<To, const From*> {
181 using ret_type = const To *; // Constant pointer arg case, return const Ty*
182};
183
184template<class To, class From> struct cast_retty_impl<To, const From*const> {
185 using ret_type = const To *; // Constant pointer arg case, return const Ty*
186};
187
188template <class To, class From>
189struct cast_retty_impl<To, std::unique_ptr<From>> {
190private:
191 using PointerType = typename cast_retty_impl<To, From *>::ret_type;
192 using ResultType = std::remove_pointer_t<PointerType>;
193
194public:
195 using ret_type = std::unique_ptr<ResultType>;
196};
197
198template<class To, class From, class SimpleFrom>
199struct cast_retty_wrap {
200 // When the simplified type and the from type are not the same, use the type
201 // simplifier to reduce the type, then reuse cast_retty_impl to get the
202 // resultant type.
203 using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
204};
205
206template<class To, class FromTy>
207struct cast_retty_wrap<To, FromTy, FromTy> {
208 // When the simplified type is equal to the from type, use it directly.
209 using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
210};
211
212template<class To, class From>
213struct cast_retty {
214 using ret_type = typename cast_retty_wrap<
215 To, From, typename simplify_type<From>::SimpleType>::ret_type;
216};
217
218// Ensure the non-simple values are converted using the simplify_type template
219// that may be specialized by smart pointers...
220//
221template<class To, class From, class SimpleFrom> struct cast_convert_val {
222 // This is not a simple type, use the template to simplify it...
223 static typename cast_retty<To, From>::ret_type doit(From &Val) {
224 return cast_convert_val<To, SimpleFrom,
30
Returning without writing to 'Val.Node'
225 typename simplify_type<SimpleFrom>::SimpleType>::doit(
226 simplify_type<From>::getSimplifiedValue(Val));
27
Calling 'simplify_type::getSimplifiedValue'
29
Returning from 'simplify_type::getSimplifiedValue'
227 }
228};
229
230template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
231 // This _is_ a simple type, just cast it.
232 static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
233 typename cast_retty<To, FromTy>::ret_type Res2
234 = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
235 return Res2;
236 }
237};
238
239template <class X> struct is_simple_type {
240 static const bool value =
241 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
242};
243
244// cast<X> - Return the argument parameter cast to the specified type. This
245// casting operator asserts that the type is correct, so it does not return null
246// on failure. It does not allow a null argument (use cast_or_null for that).
247// It is typically used like this:
248//
249// cast<Instruction>(myVal)->getParent()
250//
251template <class X, class Y>
252inline std::enable_if_t<!is_simple_type<Y>::value,
253 typename cast_retty<X, const Y>::ret_type>
254cast(const Y &Val) {
255 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 255, __extension__ __PRETTY_FUNCTION__))
;
256 return cast_convert_val<
257 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
258}
259
260template <class X, class Y>
261inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
262 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 262, __extension__ __PRETTY_FUNCTION__))
;
24
Assuming 'Val' is a 'ConstantSDNode'
25
'?' condition is true
263 return cast_convert_val<X, Y,
26
Calling 'cast_convert_val::doit'
31
Returning from 'cast_convert_val::doit'
32
Returning without writing to 'Val.Node'
264 typename simplify_type<Y>::SimpleType>::doit(Val);
265}
266
267template <class X, class Y>
268inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
269 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 269, __extension__ __PRETTY_FUNCTION__))
;
270 return cast_convert_val<X, Y*,
271 typename simplify_type<Y*>::SimpleType>::doit(Val);
272}
273
274template <class X, class Y>
275inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
276cast(std::unique_ptr<Y> &&Val) {
277 assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val.get()) &&
"cast<Ty>() argument of incompatible type!") ? void (0
) : __assert_fail ("isa<X>(Val.get()) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 277, __extension__ __PRETTY_FUNCTION__))
;
278 using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
279 return ret_type(
280 cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
281 Val.release()));
282}
283
284// cast_or_null<X> - Functionally identical to cast, except that a null value is
285// accepted.
286//
287template <class X, class Y>
288LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
289 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
290cast_or_null(const Y &Val) {
291 if (!Val)
292 return nullptr;
293 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 293, __extension__ __PRETTY_FUNCTION__))
;
294 return cast<X>(Val);
295}
296
297template <class X, class Y>
298LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
299 typename cast_retty<X, Y>::ret_type>
300cast_or_null(Y &Val) {
301 if (!Val)
302 return nullptr;
303 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 303, __extension__ __PRETTY_FUNCTION__))
;
304 return cast<X>(Val);
305}
306
307template <class X, class Y>
308LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
309cast_or_null(Y *Val) {
310 if (!Val) return nullptr;
311 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/Casting.h"
, 311, __extension__ __PRETTY_FUNCTION__))
;
312 return cast<X>(Val);
313}
314
315template <class X, class Y>
316inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
317cast_or_null(std::unique_ptr<Y> &&Val) {
318 if (!Val)
319 return nullptr;
320 return cast<X>(std::move(Val));
321}
322
323// dyn_cast<X> - Return the argument parameter cast to the specified type. This
324// casting operator returns null if the argument is of the wrong type, so it can
325// be used to test for a type as well as cast if successful. This should be
326// used in the context of an if statement like this:
327//
328// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
329//
330
331template <class X, class Y>
332LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
333 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
334dyn_cast(const Y &Val) {
335 return isa<X>(Val) ? cast<X>(Val) : nullptr;
336}
337
338template <class X, class Y>
339LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
340 return isa<X>(Val) ? cast<X>(Val) : nullptr;
21
Assuming 'Val' is a 'ConstantSDNode'
22
'?' condition is true
23
Calling 'cast<llvm::ConstantSDNode, llvm::SDValue>'
33
Returning from 'cast<llvm::ConstantSDNode, llvm::SDValue>'
34
Returning without writing to 'Val.Node'
341}
342
343template <class X, class Y>
344LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
345 return isa<X>(Val) ? cast<X>(Val) : nullptr;
346}
347
348// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
349// value is accepted.
350//
351template <class X, class Y>
352LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
353 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
354dyn_cast_or_null(const Y &Val) {
355 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
356}
357
358template <class X, class Y>
359LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
360 typename cast_retty<X, Y>::ret_type>
361dyn_cast_or_null(Y &Val) {
362 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
363}
364
365template <class X, class Y>
366LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
367dyn_cast_or_null(Y *Val) {
368 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
369}
370
371// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
372// taking ownership of the input pointer iff isa<X>(Val) is true. If the
373// cast is successful, From refers to nullptr on exit and the casted value
374// is returned. If the cast is unsuccessful, the function returns nullptr
375// and From is unchanged.
376template <class X, class Y>
377LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
378 -> decltype(cast<X>(Val)) {
379 if (!isa<X>(Val))
380 return nullptr;
381 return cast<X>(std::move(Val));
382}
383
384template <class X, class Y>
385LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
386 return unique_dyn_cast<X, Y>(Val);
387}
388
389// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
390// a null value is accepted.
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
393 -> decltype(cast<X>(Val)) {
394 if (!Val)
395 return nullptr;
396 return unique_dyn_cast<X, Y>(Val);
397}
398
399template <class X, class Y>
400LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
401 return unique_dyn_cast_or_null<X, Y>(Val);
402}
403
404} // end namespace llvm
405
406#endif // LLVM_SUPPORT_CASTING_H

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
28
Returning without writing to 'Val.Node'
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class MaskedLoadStoreSDNode;
513 friend class MaskedGatherScatterSDNode;
514
515 uint16_t : NumMemSDNodeBits;
516
517 // This storage is shared between disparate class hierarchies to hold an
518 // enumeration specific to the class hierarchy in use.
519 // LSBaseSDNode => enum ISD::MemIndexedMode
520 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
521 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
522 uint16_t AddressingMode : 3;
523 };
524 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
525
526 class LoadSDNodeBitfields {
527 friend class LoadSDNode;
528 friend class MaskedLoadSDNode;
529 friend class MaskedGatherSDNode;
530
531 uint16_t : NumLSBaseSDNodeBits;
532
533 uint16_t ExtTy : 2; // enum ISD::LoadExtType
534 uint16_t IsExpanding : 1;
535 };
536
537 class StoreSDNodeBitfields {
538 friend class StoreSDNode;
539 friend class MaskedStoreSDNode;
540 friend class MaskedScatterSDNode;
541
542 uint16_t : NumLSBaseSDNodeBits;
543
544 uint16_t IsTruncating : 1;
545 uint16_t IsCompressing : 1;
546 };
547
548 union {
549 char RawSDNodeBits[sizeof(uint16_t)];
550 SDNodeBitfields SDNodeBits;
551 ConstantSDNodeBitfields ConstantSDNodeBits;
552 MemSDNodeBitfields MemSDNodeBits;
553 LSBaseSDNodeBitfields LSBaseSDNodeBits;
554 LoadSDNodeBitfields LoadSDNodeBits;
555 StoreSDNodeBitfields StoreSDNodeBits;
556 };
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK
560
561 // RawSDNodeBits must cover the entirety of the union. This means that all of
562 // the union's members must have size <= RawSDNodeBits. We write the RHS as
563 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
564 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
565 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
566 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
567 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
568 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
570
571private:
572 friend class SelectionDAG;
573 // TODO: unfriend HandleSDNode once we fix its operand handling.
574 friend class HandleSDNode;
575
576 /// Unique id per SDNode in the DAG.
577 int NodeId = -1;
578
579 /// The values that are used by this operation.
580 SDUse *OperandList = nullptr;
581
582 /// The types of the values this node defines. SDNode's may
583 /// define multiple values simultaneously.
584 const EVT *ValueList;
585
586 /// List of uses for this SDNode.
587 SDUse *UseList = nullptr;
588
589 /// The number of entries in the Operand/Value list.
590 unsigned short NumOperands = 0;
591 unsigned short NumValues;
592
593 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
594 // original LLVM instructions.
595 // This is used for turning off scheduling, because we'll forgo
596 // the normal scheduling algorithms and output the instructions according to
597 // this ordering.
598 unsigned IROrder;
599
600 /// Source line information.
601 DebugLoc debugLoc;
602
603 /// Return a pointer to the specified value type.
604 static const EVT *getValueTypeList(EVT VT);
605
606 SDNodeFlags Flags;
607
608public:
609 /// Unique and persistent id per SDNode in the DAG.
610 /// Used for debug printing.
611 uint16_t PersistentId;
612
613 //===--------------------------------------------------------------------===//
614 // Accessors
615 //
616
617 /// Return the SelectionDAG opcode value for this node. For
618 /// pre-isel nodes (those for which isMachineOpcode returns false), these
619 /// are the opcode values in the ISD and <target>ISD namespaces. For
620 /// post-isel opcodes, see getMachineOpcode.
621 unsigned getOpcode() const { return (unsigned short)NodeType; }
622
623 /// Test if this node has a target-specific opcode (in the
624 /// \<target\>ISD namespace).
625 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
626
627 /// Test if this node has a target-specific opcode that may raise
628 /// FP exceptions (in the \<target\>ISD namespace and greater than
629 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
630 /// opcode are currently automatically considered to possibly raise
631 /// FP exceptions as well.
632 bool isTargetStrictFPOpcode() const {
633 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
634 }
635
636 /// Test if this node has a target-specific
637 /// memory-referencing opcode (in the \<target\>ISD namespace and
638 /// greater than FIRST_TARGET_MEMORY_OPCODE).
639 bool isTargetMemoryOpcode() const {
640 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
641 }
642
643 /// Return true if the type of the node type undefined.
644 bool isUndef() const { return NodeType == ISD::UNDEF; }
645
646 /// Test if this node is a memory intrinsic (with valid pointer information).
647 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
648 /// non-memory intrinsics (with chains) that are not really instances of
649 /// MemSDNode. For such nodes, we need some extra state to determine the
650 /// proper classof relationship.
651 bool isMemIntrinsic() const {
652 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
653 NodeType == ISD::INTRINSIC_VOID) &&
654 SDNodeBits.IsMemIntrinsic;
655 }
656
657 /// Test if this node is a strict floating point pseudo-op.
658 bool isStrictFPOpcode() {
659 switch (NodeType) {
660 default:
661 return false;
662 case ISD::STRICT_FP16_TO_FP:
663 case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
665 case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
667 return true;
668 }
669 }
670
671 /// Test if this node has a post-isel opcode, directly
672 /// corresponding to a MachineInstr opcode.
673 bool isMachineOpcode() const { return NodeType < 0; }
674
675 /// This may only be called if isMachineOpcode returns
676 /// true. It returns the MachineInstr opcode value that the node's opcode
677 /// corresponds to.
678 unsigned getMachineOpcode() const {
679 assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 679, __extension__ __PRETTY_FUNCTION__))
;
680 return ~NodeType;
681 }
682
683 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
684 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
685
686 bool isDivergent() const { return SDNodeBits.IsDivergent; }
687
688 /// Return true if there are no uses of this node.
689 bool use_empty() const { return UseList == nullptr; }
690
691 /// Return true if there is exactly one use of this node.
692 bool hasOneUse() const { return hasSingleElement(uses()); }
693
694 /// Return the number of uses of this node. This method takes
695 /// time proportional to the number of uses.
696 size_t use_size() const { return std::distance(use_begin(), use_end()); }
697
698 /// Return the unique node id.
699 int getNodeId() const { return NodeId; }
700
701 /// Set unique node id.
702 void setNodeId(int Id) { NodeId = Id; }
703
704 /// Return the node ordering.
705 unsigned getIROrder() const { return IROrder; }
706
707 /// Set the node ordering.
708 void setIROrder(unsigned Order) { IROrder = Order; }
709
710 /// Return the source location info.
711 const DebugLoc &getDebugLoc() const { return debugLoc; }
712
713 /// Set source location info. Try to avoid this, putting
714 /// it in the constructor is preferable.
715 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
716
717 /// This class provides iterator support for SDUse
718 /// operands that use a specific SDNode.
719 class use_iterator {
720 friend class SDNode;
721
722 SDUse *Op = nullptr;
723
724 explicit use_iterator(SDUse *op) : Op(op) {}
725
726 public:
727 using iterator_category = std::forward_iterator_tag;
728 using value_type = SDUse;
729 using difference_type = std::ptrdiff_t;
730 using pointer = value_type *;
731 using reference = value_type &;
732
733 use_iterator() = default;
734 use_iterator(const use_iterator &I) : Op(I.Op) {}
735
736 bool operator==(const use_iterator &x) const {
737 return Op == x.Op;
738 }
739 bool operator!=(const use_iterator &x) const {
740 return !operator==(x);
741 }
742
743 /// Return true if this iterator is at the end of uses list.
744 bool atEnd() const { return Op == nullptr; }
745
746 // Iterator traversal: forward iteration only.
747 use_iterator &operator++() { // Preincrement
748 assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 748, __extension__ __PRETTY_FUNCTION__))
;
749 Op = Op->getNext();
750 return *this;
751 }
752
753 use_iterator operator++(int) { // Postincrement
754 use_iterator tmp = *this; ++*this; return tmp;
755 }
756
757 /// Retrieve a pointer to the current user node.
758 SDNode *operator*() const {
759 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 759, __extension__ __PRETTY_FUNCTION__))
;
760 return Op->getUser();
761 }
762
763 SDNode *operator->() const { return operator*(); }
764
765 SDUse &getUse() const { return *Op; }
766
767 /// Retrieve the operand # of this use in its user.
768 unsigned getOperandNo() const {
769 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __extension__ __PRETTY_FUNCTION__))
;
770 return (unsigned)(Op - Op->getUser()->OperandList);
771 }
772 };
773
774 /// Provide iteration support to walk over all uses of an SDNode.
775 use_iterator use_begin() const {
776 return use_iterator(UseList);
777 }
778
779 static use_iterator use_end() { return use_iterator(nullptr); }
780
781 inline iterator_range<use_iterator> uses() {
782 return make_range(use_begin(), use_end());
783 }
784 inline iterator_range<use_iterator> uses() const {
785 return make_range(use_begin(), use_end());
786 }
787
788 /// Return true if there are exactly NUSES uses of the indicated value.
789 /// This method ignores uses of other values defined by this operation.
790 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
791
792 /// Return true if there are any use of the indicated value.
793 /// This method ignores uses of other values defined by this operation.
794 bool hasAnyUseOfValue(unsigned Value) const;
795
796 /// Return true if this node is the only use of N.
797 bool isOnlyUserOf(const SDNode *N) const;
798
799 /// Return true if this node is an operand of N.
800 bool isOperandOf(const SDNode *N) const;
801
802 /// Return true if this node is a predecessor of N.
803 /// NOTE: Implemented on top of hasPredecessor and every bit as
804 /// expensive. Use carefully.
805 bool isPredecessorOf(const SDNode *N) const {
806 return N->hasPredecessor(this);
807 }
808
809 /// Return true if N is a predecessor of this node.
810 /// N is either an operand of this node, or can be reached by recursively
811 /// traversing up the operands.
812 /// NOTE: This is an expensive method. Use it carefully.
813 bool hasPredecessor(const SDNode *N) const;
814
815 /// Returns true if N is a predecessor of any node in Worklist. This
816 /// helper keeps Visited and Worklist sets externally to allow unions
817 /// searches to be performed in parallel, caching of results across
818 /// queries and incremental addition to Worklist. Stops early if N is
819 /// found but will resume. Remember to clear Visited and Worklists
820 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
821 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
822 /// topologically ordered (Operands have strictly smaller node id) and search
823 /// can be pruned leveraging this.
824 static bool hasPredecessorHelper(const SDNode *N,
825 SmallPtrSetImpl<const SDNode *> &Visited,
826 SmallVectorImpl<const SDNode *> &Worklist,
827 unsigned int MaxSteps = 0,
828 bool TopologicalPrune = false) {
829 SmallVector<const SDNode *, 8> DeferredNodes;
830 if (Visited.count(N))
831 return true;
832
833 // Node Id's are assigned in three places: As a topological
834 // ordering (> 0), during legalization (results in values set to
835 // 0), new nodes (set to -1). If N has a topolgical id then we
836 // know that all nodes with ids smaller than it cannot be
837 // successors and we need not check them. Filter out all node
838 // that can't be matches. We add them to the worklist before exit
839 // in case of multiple calls. Note that during selection the topological id
840 // may be violated if a node's predecessor is selected before it. We mark
841 // this at selection negating the id of unselected successors and
842 // restricting topological pruning to positive ids.
843
844 int NId = N->getNodeId();
845 // If we Invalidated the Id, reconstruct original NId.
846 if (NId < -1)
847 NId = -(NId + 1);
848
849 bool Found = false;
850 while (!Worklist.empty()) {
851 const SDNode *M = Worklist.pop_back_val();
852 int MId = M->getNodeId();
853 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
854 (MId > 0) && (MId < NId)) {
855 DeferredNodes.push_back(M);
856 continue;
857 }
858 for (const SDValue &OpV : M->op_values()) {
859 SDNode *Op = OpV.getNode();
860 if (Visited.insert(Op).second)
861 Worklist.push_back(Op);
862 if (Op == N)
863 Found = true;
864 }
865 if (Found)
866 break;
867 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
868 break;
869 }
870 // Push deferred nodes back on worklist.
871 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
872 // If we bailed early, conservatively return found.
873 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
874 return true;
875 return Found;
876 }
877
878 /// Return true if all the users of N are contained in Nodes.
879 /// NOTE: Requires at least one match, but doesn't require them all.
880 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
881
882 /// Return the number of values used by this operation.
883 unsigned getNumOperands() const { return NumOperands; }
884
885 /// Return the maximum number of operands that a SDNode can hold.
886 static constexpr size_t getMaxNumOperands() {
887 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
888 }
889
890 /// Helper method returns the integer value of a ConstantSDNode operand.
891 inline uint64_t getConstantOperandVal(unsigned Num) const;
892
893 /// Helper method returns the APInt of a ConstantSDNode operand.
894 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
895
896 const SDValue &getOperand(unsigned Num) const {
897 assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 897, __extension__ __PRETTY_FUNCTION__))
;
898 return OperandList[Num];
899 }
900
901 using op_iterator = SDUse *;
902
903 op_iterator op_begin() const { return OperandList; }
904 op_iterator op_end() const { return OperandList+NumOperands; }
905 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
906
907 /// Iterator for directly iterating over the operand SDValue's.
908 struct value_op_iterator
909 : iterator_adaptor_base<value_op_iterator, op_iterator,
910 std::random_access_iterator_tag, SDValue,
911 ptrdiff_t, value_op_iterator *,
912 value_op_iterator *> {
913 explicit value_op_iterator(SDUse *U = nullptr)
914 : iterator_adaptor_base(U) {}
915
916 const SDValue &operator*() const { return I->get(); }
917 };
918
919 iterator_range<value_op_iterator> op_values() const {
920 return make_range(value_op_iterator(op_begin()),
921 value_op_iterator(op_end()));
922 }
923
924 SDVTList getVTList() const {
925 SDVTList X = { ValueList, NumValues };
926 return X;
927 }
928
929 /// If this node has a glue operand, return the node
930 /// to which the glue operand points. Otherwise return NULL.
931 SDNode *getGluedNode() const {
932 if (getNumOperands() != 0 &&
933 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
934 return getOperand(getNumOperands()-1).getNode();
935 return nullptr;
936 }
937
938 /// If this node has a glue value with a user, return
939 /// the user (there is at most one). Otherwise return NULL.
940 SDNode *getGluedUser() const {
941 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
942 if (UI.getUse().get().getValueType() == MVT::Glue)
943 return *UI;
944 return nullptr;
945 }
946
947 SDNodeFlags getFlags() const { return Flags; }
948 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
949
950 /// Clear any flags in this node that aren't also set in Flags.
951 /// If Flags is not in a defined state then this has no effect.
952 void intersectFlagsWith(const SDNodeFlags Flags);
953
954 /// Return the number of values defined/returned by this operator.
955 unsigned getNumValues() const { return NumValues; }
956
957 /// Return the type of a specified result.
958 EVT getValueType(unsigned ResNo) const {
959 assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 959, __extension__ __PRETTY_FUNCTION__))
;
960 return ValueList[ResNo];
961 }
962
963 /// Return the type of a specified result as a simple type.
964 MVT getSimpleValueType(unsigned ResNo) const {
965 return getValueType(ResNo).getSimpleVT();
966 }
967
968 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
969 ///
970 /// If the value type is a scalable vector type, the scalable property will
971 /// be set and the runtime size will be a positive integer multiple of the
972 /// base size.
973 TypeSize getValueSizeInBits(unsigned ResNo) const {
974 return getValueType(ResNo).getSizeInBits();
975 }
976
977 using value_iterator = const EVT *;
978
979 value_iterator value_begin() const { return ValueList; }
980 value_iterator value_end() const { return ValueList+NumValues; }
981 iterator_range<value_iterator> values() const {
982 return llvm::make_range(value_begin(), value_end());
983 }
984
985 /// Return the opcode of this operation for printing.
986 std::string getOperationName(const SelectionDAG *G = nullptr) const;
987 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
988 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
989 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
990 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
991 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
992
993 /// Print a SelectionDAG node and all children down to
994 /// the leaves. The given SelectionDAG allows target-specific nodes
995 /// to be printed in human-readable form. Unlike printr, this will
996 /// print the whole DAG, including children that appear multiple
997 /// times.
998 ///
999 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and children up to
1002 /// depth "depth." The given SelectionDAG allows target-specific
1003 /// nodes to be printed in human-readable form. Unlike printr, this
1004 /// will print children that appear multiple times wherever they are
1005 /// used.
1006 ///
1007 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1008 unsigned depth = 100) const;
1009
1010 /// Dump this node, for debugging.
1011 void dump() const;
1012
1013 /// Dump (recursively) this node and its use-def subgraph.
1014 void dumpr() const;
1015
1016 /// Dump this node, for debugging.
1017 /// The given SelectionDAG allows target-specific nodes to be printed
1018 /// in human-readable form.
1019 void dump(const SelectionDAG *G) const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 /// The given SelectionDAG allows target-specific nodes to be printed
1023 /// in human-readable form.
1024 void dumpr(const SelectionDAG *G) const;
1025
1026 /// printrFull to dbgs(). The given SelectionDAG allows
1027 /// target-specific nodes to be printed in human-readable form.
1028 /// Unlike dumpr, this will print the whole DAG, including children
1029 /// that appear multiple times.
1030 void dumprFull(const SelectionDAG *G = nullptr) const;
1031
1032 /// printrWithDepth to dbgs(). The given
1033 /// SelectionDAG allows target-specific nodes to be printed in
1034 /// human-readable form. Unlike dumpr, this will print children
1035 /// that appear multiple times wherever they are used.
1036 ///
1037 void dumprWithDepth(const SelectionDAG *G = nullptr,
1038 unsigned depth = 100) const;
1039
1040 /// Gather unique data for the node.
1041 void Profile(FoldingSetNodeID &ID) const;
1042
1043 /// This method should only be used by the SDUse class.
1044 void addUse(SDUse &U) { U.addToList(&UseList); }
1045
1046protected:
1047 static SDVTList getSDVTList(EVT VT) {
1048 SDVTList Ret = { getValueTypeList(VT), 1 };
1049 return Ret;
1050 }
1051
1052 /// Create an SDNode.
1053 ///
1054 /// SDNodes are created without any operands, and never own the operand
1055 /// storage. To add operands, see SelectionDAG::createOperands.
1056 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1057 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1058 IROrder(Order), debugLoc(std::move(dl)) {
1059 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1060 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
"Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1060, __extension__ __PRETTY_FUNCTION__))
;
1061 assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
1062 "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
;
1063 }
1064
1065 /// Release the operands and set this node to have zero operands.
1066 void DropOperands();
1067};
1068
1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
1080 DebugLoc DL;
1081 int IROrder = 0;
1082
1083public:
1084 SDLoc() = default;
1085 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1086 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1087 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1088 assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1088, __extension__ __PRETTY_FUNCTION__))
;
1089 if (I)
1090 DL = I->getDebugLoc();
1091 }
1092
1093 unsigned getIROrder() const { return IROrder; }
1094 const DebugLoc &getDebugLoc() const { return DL; }
1095};
1096
1097// Define inline functions from the SDValue class.
1098
1099inline SDValue::SDValue(SDNode *node, unsigned resno)
1100 : Node(node), ResNo(resno) {
1101 // Explicitly check for !ResNo to avoid use-after-free, because there are
1102 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1103 // combines.
1104 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
1105 "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
;
1106 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1106, __extension__ __PRETTY_FUNCTION__))
;
1107}
1108
1109inline unsigned SDValue::getOpcode() const {
1110 return Node->getOpcode();
38
Called C++ object pointer is null
1111}
1112
1113inline EVT SDValue::getValueType() const {
1114 return Node->getValueType(ResNo);
1115}
1116
1117inline unsigned SDValue::getNumOperands() const {
1118 return Node->getNumOperands();
1119}
1120
1121inline const SDValue &SDValue::getOperand(unsigned i) const {
1122 return Node->getOperand(i);
1123}
1124
1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1126 return Node->getConstantOperandVal(i);
1127}
1128
1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1130 return Node->getConstantOperandAPInt(i);
1131}
1132
1133inline bool SDValue::isTargetOpcode() const {
1134 return Node->isTargetOpcode();
1135}
1136
1137inline bool SDValue::isTargetMemoryOpcode() const {
1138 return Node->isTargetMemoryOpcode();
1139}
1140
1141inline bool SDValue::isMachineOpcode() const {
1142 return Node->isMachineOpcode();
1143}
1144
1145inline unsigned SDValue::getMachineOpcode() const {
1146 return Node->getMachineOpcode();
1147}
1148
1149inline bool SDValue::isUndef() const {
1150 return Node->isUndef();
1151}
1152
1153inline bool SDValue::use_empty() const {
1154 return !Node->hasAnyUseOfValue(ResNo);
1155}
1156
1157inline bool SDValue::hasOneUse() const {
1158 return Node->hasNUsesOfValue(1, ResNo);
1159}
1160
1161inline const DebugLoc &SDValue::getDebugLoc() const {
1162 return Node->getDebugLoc();
1163}
1164
1165inline void SDValue::dump() const {
1166 return Node->dump();
1167}
1168
1169inline void SDValue::dump(const SelectionDAG *G) const {
1170 return Node->dump(G);
1171}
1172
1173inline void SDValue::dumpr() const {
1174 return Node->dumpr();
1175}
1176
1177inline void SDValue::dumpr(const SelectionDAG *G) const {
1178 return Node->dumpr(G);
1179}
1180
1181// Define inline functions from the SDUse class.
1182
1183inline void SDUse::set(const SDValue &V) {
1184 if (Val.getNode()) removeFromList();
1185 Val = V;
1186 if (V.getNode()) V.getNode()->addUse(*this);
1187}
1188
1189inline void SDUse::setInitial(const SDValue &V) {
1190 Val = V;
1191 V.getNode()->addUse(*this);
1192}
1193
1194inline void SDUse::setNode(SDNode *N) {
1195 if (Val.getNode()) removeFromList();
1196 Val.setNode(N);
1197 if (N) N->addUse(*this);
1198}
1199
1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand. This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
1205 SDUse Op;
1206
1207public:
1208 explicit HandleSDNode(SDValue X)
1209 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1210 // HandleSDNodes are never inserted into the DAG, so they won't be
1211 // auto-numbered. Use ID 65535 as a sentinel.
1212 PersistentId = 0xffff;
1213
1214 // Manually set up the operand list. This node type is special in that it's
1215 // always stack allocated and SelectionDAG does not manage its operands.
1216 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1217 // be so special.
1218 Op.setUser(this);
1219 Op.setInitial(X);
1220 NumOperands = 1;
1221 OperandList = &Op;
1222 }
1223 ~HandleSDNode();
1224
1225 const SDValue &getValue() const { return Op; }
1226};
1227
1228class AddrSpaceCastSDNode : public SDNode {
1229private:
1230 unsigned SrcAddrSpace;
1231 unsigned DestAddrSpace;
1232
1233public:
1234 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1235 unsigned SrcAS, unsigned DestAS);
1236
1237 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1238 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1239
1240 static bool classof(const SDNode *N) {
1241 return N->getOpcode() == ISD::ADDRSPACECAST;
1242 }
1243};
1244
1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
1248 // VT of in-memory value.
1249 EVT MemoryVT;
1250
1251protected:
1252 /// Memory reference information.
1253 MachineMemOperand *MMO;
1254
1255public:
1256 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1257 EVT memvt, MachineMemOperand *MMO);
1258
1259 bool readMem() const { return MMO->isLoad(); }
1260 bool writeMem() const { return MMO->isStore(); }
1261
1262 /// Returns alignment and volatility of the memory access
1263 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1264 Align getAlign() const { return MMO->getAlign(); }
1265 // FIXME: Remove once transition to getAlign is over.
1266 unsigned getAlignment() const { return MMO->getAlign().value(); }
1267
1268 /// Return the SubclassData value, without HasDebugValue. This contains an
1269 /// encoding of the volatile flag, as well as bits used by subclasses. This
1270 /// function should only be used to compute a FoldingSetNodeID value.
1271 /// The HasDebugValue bit is masked out because CSE map needs to match
1272 /// nodes with debug info with nodes without debug info. Same is about
1273 /// isDivergent bit.
1274 unsigned getRawSubclassData() const {
1275 uint16_t Data;
1276 union {
1277 char RawSDNodeBits[sizeof(uint16_t)];
1278 SDNodeBitfields SDNodeBits;
1279 };
1280 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1281 SDNodeBits.HasDebugValue = 0;
1282 SDNodeBits.IsDivergent = false;
1283 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1284 return Data;
1285 }
1286
1287 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1288 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1289 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1290 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1291
1292 // Returns the offset from the location of the access.
1293 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1294
1295 /// Returns the AA info that describes the dereference.
1296 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1297
1298 /// Returns the Ranges that describes the dereference.
1299 const MDNode *getRanges() const { return MMO->getRanges(); }
1300
1301 /// Returns the synchronization scope ID for this memory operation.
1302 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1303
1304 /// Return the atomic ordering requirements for this memory operation. For
1305 /// cmpxchg atomic operations, return the atomic ordering requirements when
1306 /// store occurs.
1307 AtomicOrdering getSuccessOrdering() const {
1308 return MMO->getSuccessOrdering();
1309 }
1310
1311 /// Return a single atomic ordering that is at least as strong as both the
1312 /// success and failure orderings for an atomic operation. (For operations
1313 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1314 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1315
1316 /// Return true if the memory operation ordering is Unordered or higher.
1317 bool isAtomic() const { return MMO->isAtomic(); }
1318
1319 /// Returns true if the memory operation doesn't imply any ordering
1320 /// constraints on surrounding memory operations beyond the normal memory
1321 /// aliasing rules.
1322 bool isUnordered() const { return MMO->isUnordered(); }
1323
1324 /// Returns true if the memory operation is neither atomic or volatile.
1325 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1326
1327 /// Return the type of the in-memory value.
1328 EVT getMemoryVT() const { return MemoryVT; }
1329
1330 /// Return a MachineMemOperand object describing the memory
1331 /// reference performed by operation.
1332 MachineMemOperand *getMemOperand() const { return MMO; }
1333
1334 const MachinePointerInfo &getPointerInfo() const {
1335 return MMO->getPointerInfo();
1336 }
1337
1338 /// Return the address space for the associated pointer
1339 unsigned getAddressSpace() const {
1340 return getPointerInfo().getAddrSpace();
1341 }
1342
1343 /// Update this MemSDNode's MachineMemOperand information
1344 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1345 /// This must only be used when the new alignment applies to all users of
1346 /// this MachineMemOperand.
1347 void refineAlignment(const MachineMemOperand *NewMMO) {
1348 MMO->refineAlignment(NewMMO);
1349 }
1350
1351 const SDValue &getChain() const { return getOperand(0); }
1352
1353 const SDValue &getBasePtr() const {
1354 switch (getOpcode()) {
1355 case ISD::STORE:
1356 case ISD::MSTORE:
1357 return getOperand(2);
1358 case ISD::MGATHER:
1359 case ISD::MSCATTER:
1360 return getOperand(3);
1361 default:
1362 return getOperand(1);
1363 }
1364 }
1365
1366 // Methods to support isa and dyn_cast
1367 static bool classof(const SDNode *N) {
1368 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1369 // with either an intrinsic or a target opcode.
1370 switch (N->getOpcode()) {
1371 case ISD::LOAD:
1372 case ISD::STORE:
1373 case ISD::PREFETCH:
1374 case ISD::ATOMIC_CMP_SWAP:
1375 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1376 case ISD::ATOMIC_SWAP:
1377 case ISD::ATOMIC_LOAD_ADD:
1378 case ISD::ATOMIC_LOAD_SUB:
1379 case ISD::ATOMIC_LOAD_AND:
1380 case ISD::ATOMIC_LOAD_CLR:
1381 case ISD::ATOMIC_LOAD_OR:
1382 case ISD::ATOMIC_LOAD_XOR:
1383 case ISD::ATOMIC_LOAD_NAND:
1384 case ISD::ATOMIC_LOAD_MIN:
1385 case ISD::ATOMIC_LOAD_MAX:
1386 case ISD::ATOMIC_LOAD_UMIN:
1387 case ISD::ATOMIC_LOAD_UMAX:
1388 case ISD::ATOMIC_LOAD_FADD:
1389 case ISD::ATOMIC_LOAD_FSUB:
1390 case ISD::ATOMIC_LOAD:
1391 case ISD::ATOMIC_STORE:
1392 case ISD::MLOAD:
1393 case ISD::MSTORE:
1394 case ISD::MGATHER:
1395 case ISD::MSCATTER:
1396 return true;
1397 default:
1398 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1399 }
1400 }
1401};
1402
1403/// This is an SDNode representing atomic operations.
1404class AtomicSDNode : public MemSDNode {
1405public:
1406 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1407 EVT MemVT, MachineMemOperand *MMO)
1408 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1409 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
1410 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
;
1411 }
1412
1413 const SDValue &getBasePtr() const { return getOperand(1); }
1414 const SDValue &getVal() const { return getOperand(2); }
1415
1416 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1417 /// otherwise.
1418 bool isCompareAndSwap() const {
1419 unsigned Op = getOpcode();
1420 return Op == ISD::ATOMIC_CMP_SWAP ||
1421 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1422 }
1423
1424 /// For cmpxchg atomic operations, return the atomic ordering requirements
1425 /// when store does not occur.
1426 AtomicOrdering getFailureOrdering() const {
1427 assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1427, __extension__ __PRETTY_FUNCTION__))
;
1428 return MMO->getFailureOrdering();
1429 }
1430
1431 // Methods to support isa and dyn_cast
1432 static bool classof(const SDNode *N) {
1433 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1434 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1435 N->getOpcode() == ISD::ATOMIC_SWAP ||
1436 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1437 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1438 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1439 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1440 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1441 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1442 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1443 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1444 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1445 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1446 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1447 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1448 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1449 N->getOpcode() == ISD::ATOMIC_LOAD ||
1450 N->getOpcode() == ISD::ATOMIC_STORE;
1451 }
1452};
1453
1454/// This SDNode is used for target intrinsics that touch
1455/// memory and need an associated MachineMemOperand. Its opcode may be
1456/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1457/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1458class MemIntrinsicSDNode : public MemSDNode {
1459public:
1460 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1461 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1462 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1463 SDNodeBits.IsMemIntrinsic = true;
1464 }
1465
1466 // Methods to support isa and dyn_cast
1467 static bool classof(const SDNode *N) {
1468 // We lower some target intrinsics to their target opcode
1469 // early a node with a target opcode can be of this class
1470 return N->isMemIntrinsic() ||
1471 N->getOpcode() == ISD::PREFETCH ||
1472 N->isTargetMemoryOpcode();
1473 }
1474};
1475
1476/// This SDNode is used to implement the code generator
1477/// support for the llvm IR shufflevector instruction. It combines elements
1478/// from two input vectors into a new input vector, with the selection and
1479/// ordering of elements determined by an array of integers, referred to as
1480/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1481/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1482/// An index of -1 is treated as undef, such that the code generator may put
1483/// any value in the corresponding element of the result.
1484class ShuffleVectorSDNode : public SDNode {
1485 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1486 // is freed when the SelectionDAG object is destroyed.
1487 const int *Mask;
1488
1489protected:
1490 friend class SelectionDAG;
1491
1492 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1493 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1494
1495public:
1496 ArrayRef<int> getMask() const {
1497 EVT VT = getValueType(0);
1498 return makeArrayRef(Mask, VT.getVectorNumElements());
1499 }
1500
1501 int getMaskElt(unsigned Idx) const {
1502 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1502, __extension__ __PRETTY_FUNCTION__))
;
1503 return Mask[Idx];
1504 }
1505
1506 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1507
1508 int getSplatIndex() const {
1509 assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1509, __extension__ __PRETTY_FUNCTION__))
;
1510 EVT VT = getValueType(0);
1511 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1512 if (Mask[i] >= 0)
1513 return Mask[i];
1514
1515 // We can choose any index value here and be correct because all elements
1516 // are undefined. Return 0 for better potential for callers to simplify.
1517 return 0;
1518 }
1519
1520 static bool isSplatMask(const int *Mask, EVT VT);
1521
1522 /// Change values in a shuffle permute mask assuming
1523 /// the two vector operands have swapped position.
1524 static void commuteMask(MutableArrayRef<int> Mask) {
1525 unsigned NumElems = Mask.size();
1526 for (unsigned i = 0; i != NumElems; ++i) {
1527 int idx = Mask[i];
1528 if (idx < 0)
1529 continue;
1530 else if (idx < (int)NumElems)
1531 Mask[i] = idx + NumElems;
1532 else
1533 Mask[i] = idx - NumElems;
1534 }
1535 }
1536
1537 static bool classof(const SDNode *N) {
1538 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1539 }
1540};
1541
1542class ConstantSDNode : public SDNode {
1543 friend class SelectionDAG;
1544
1545 const ConstantInt *Value;
1546
1547 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1548 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1549 getSDVTList(VT)),
1550 Value(val) {
1551 ConstantSDNodeBits.IsOpaque = isOpaque;
1552 }
1553
1554public:
1555 const ConstantInt *getConstantIntValue() const { return Value; }
1556 const APInt &getAPIntValue() const { return Value->getValue(); }
1557 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1558 int64_t getSExtValue() const { return Value->getSExtValue(); }
1559 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1560 return Value->getLimitedValue(Limit);
1561 }
1562 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1563 Align getAlignValue() const { return Value->getAlignValue(); }
1564
1565 bool isOne() const { return Value->isOne(); }
1566 bool isNullValue() const { return Value->isZero(); }
1567 bool isAllOnesValue() const { return Value->isMinusOne(); }
1568 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1569 bool isMinSignedValue() const { return Value->isMinValue(true); }
1570
1571 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1572
1573 static bool classof(const SDNode *N) {
1574 return N->getOpcode() == ISD::Constant ||
1575 N->getOpcode() == ISD::TargetConstant;
1576 }
1577};
1578
1579uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1580 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1581}
1582
1583const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1584 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1585}
1586
1587class ConstantFPSDNode : public SDNode {
1588 friend class SelectionDAG;
1589
1590 const ConstantFP *Value;
1591
1592 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1593 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1594 DebugLoc(), getSDVTList(VT)),
1595 Value(val) {}
1596
1597public:
1598 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1599 const ConstantFP *getConstantFPValue() const { return Value; }
1600
1601 /// Return true if the value is positive or negative zero.
1602 bool isZero() const { return Value->isZero(); }
1603
1604 /// Return true if the value is a NaN.
1605 bool isNaN() const { return Value->isNaN(); }
1606
1607 /// Return true if the value is an infinity
1608 bool isInfinity() const { return Value->isInfinity(); }
1609
1610 /// Return true if the value is negative.
1611 bool isNegative() const { return Value->isNegative(); }
1612
1613 /// We don't rely on operator== working on double values, as
1614 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1615 /// As such, this method can be used to do an exact bit-for-bit comparison of
1616 /// two floating point values.
1617
1618 /// We leave the version with the double argument here because it's just so
1619 /// convenient to write "2.0" and the like. Without this function we'd
1620 /// have to duplicate its logic everywhere it's called.
1621 bool isExactlyValue(double V) const {
1622 return Value->getValueAPF().isExactlyValue(V);
1623 }
1624 bool isExactlyValue(const APFloat& V) const;
1625
1626 static bool isValueValidForType(EVT VT, const APFloat& Val);
1627
1628 static bool classof(const SDNode *N) {
1629 return N->getOpcode() == ISD::ConstantFP ||
1630 N->getOpcode() == ISD::TargetConstantFP;
1631 }
1632};
1633
1634/// Returns true if \p V is a constant integer zero.
1635bool isNullConstant(SDValue V);
1636
1637/// Returns true if \p V is an FP constant with a value of positive zero.
1638bool isNullFPConstant(SDValue V);
1639
1640/// Returns true if \p V is an integer constant with all bits set.
1641bool isAllOnesConstant(SDValue V);
1642
1643/// Returns true if \p V is a constant integer one.
1644bool isOneConstant(SDValue V);
1645
1646/// Return the non-bitcasted source operand of \p V if it exists.
1647/// If \p V is not a bitcasted value, it is returned as-is.
1648SDValue peekThroughBitcasts(SDValue V);
1649
1650/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1651/// If \p V is not a bitcasted one-use value, it is returned as-is.
1652SDValue peekThroughOneUseBitcasts(SDValue V);
1653
1654/// Return the non-extracted vector source operand of \p V if it exists.
1655/// If \p V is not an extracted subvector, it is returned as-is.
1656SDValue peekThroughExtractSubvectors(SDValue V);
1657
1658/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1659/// constant is canonicalized to be operand 1.
1660bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1661
1662/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1663ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1664 bool AllowTruncation = false);
1665
1666/// Returns the SDNode if it is a demanded constant splat BuildVector or
1667/// constant int.
1668ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1669 bool AllowUndefs = false,
1670 bool AllowTruncation = false);
1671
1672/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1673ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1674
1675/// Returns the SDNode if it is a demanded constant splat BuildVector or
1676/// constant float.
1677ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1678 bool AllowUndefs = false);
1679
1680/// Return true if the value is a constant 0 integer or a splatted vector of
1681/// a constant 0 integer (with no undefs by default).
1682/// Build vector implicit truncation is not an issue for null values.
1683bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1684
1685/// Return true if the value is a constant 1 integer or a splatted vector of a
1686/// constant 1 integer (with no undefs).
1687/// Does not permit build vector implicit truncation.
1688bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1689
1690/// Return true if the value is a constant -1 integer or a splatted vector of a
1691/// constant -1 integer (with no undefs).
1692/// Does not permit build vector implicit truncation.
1693bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1694
1695/// Return true if \p V is either a integer or FP constant.
1696inline bool isIntOrFPConstant(SDValue V) {
1697 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1698}
1699
1700class GlobalAddressSDNode : public SDNode {
1701 friend class SelectionDAG;
1702
1703 const GlobalValue *TheGlobal;
1704 int64_t Offset;
1705 unsigned TargetFlags;
1706
1707 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1708 const GlobalValue *GA, EVT VT, int64_t o,
1709 unsigned TF);
1710
1711public:
1712 const GlobalValue *getGlobal() const { return TheGlobal; }
1713 int64_t getOffset() const { return Offset; }
1714 unsigned getTargetFlags() const { return TargetFlags; }
1715 // Return the address space this GlobalAddress belongs to.
1716 unsigned getAddressSpace() const;
1717
1718 static bool classof(const SDNode *N) {
1719 return N->getOpcode() == ISD::GlobalAddress ||
1720 N->getOpcode() == ISD::TargetGlobalAddress ||
1721 N->getOpcode() == ISD::GlobalTLSAddress ||
1722 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1723 }
1724};
1725
1726class FrameIndexSDNode : public SDNode {
1727 friend class SelectionDAG;
1728
1729 int FI;
1730
1731 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1732 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1733 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1734 }
1735
1736public:
1737 int getIndex() const { return FI; }
1738
1739 static bool classof(const SDNode *N) {
1740 return N->getOpcode() == ISD::FrameIndex ||
1741 N->getOpcode() == ISD::TargetFrameIndex;
1742 }
1743};
1744
1745/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1746/// the offet and size that are started/ended in the underlying FrameIndex.
1747class LifetimeSDNode : public SDNode {
1748 friend class SelectionDAG;
1749 int64_t Size;
1750 int64_t Offset; // -1 if offset is unknown.
1751
1752 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1753 SDVTList VTs, int64_t Size, int64_t Offset)
1754 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1755public:
1756 int64_t getFrameIndex() const {
1757 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1758 }
1759
1760 bool hasOffset() const { return Offset >= 0; }
1761 int64_t getOffset() const {
1762 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1762, __extension__ __PRETTY_FUNCTION__))
;
1763 return Offset;
1764 }
1765 int64_t getSize() const {
1766 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1766, __extension__ __PRETTY_FUNCTION__))
;
1767 return Size;
1768 }
1769
1770 // Methods to support isa and dyn_cast
1771 static bool classof(const SDNode *N) {
1772 return N->getOpcode() == ISD::LIFETIME_START ||
1773 N->getOpcode() == ISD::LIFETIME_END;
1774 }
1775};
1776
1777/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1778/// the index of the basic block being probed. A pseudo probe serves as a place
1779/// holder and will be removed at the end of compilation. It does not have any
1780/// operand because we do not want the instruction selection to deal with any.
1781class PseudoProbeSDNode : public SDNode {
1782 friend class SelectionDAG;
1783 uint64_t Guid;
1784 uint64_t Index;
1785 uint32_t Attributes;
1786
1787 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1788 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1789 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1790 Attributes(Attr) {}
1791
1792public:
1793 uint64_t getGuid() const { return Guid; }
1794 uint64_t getIndex() const { return Index; }
1795 uint32_t getAttributes() const { return Attributes; }
1796
1797 // Methods to support isa and dyn_cast
1798 static bool classof(const SDNode *N) {
1799 return N->getOpcode() == ISD::PSEUDO_PROBE;
1800 }
1801};
1802
1803class JumpTableSDNode : public SDNode {
1804 friend class SelectionDAG;
1805
1806 int JTI;
1807 unsigned TargetFlags;
1808
1809 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1810 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1811 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1812 }
1813
1814public:
1815 int getIndex() const { return JTI; }
1816 unsigned getTargetFlags() const { return TargetFlags; }
1817
1818 static bool classof(const SDNode *N) {
1819 return N->getOpcode() == ISD::JumpTable ||
1820 N->getOpcode() == ISD::TargetJumpTable;
1821 }
1822};
1823
1824class ConstantPoolSDNode : public SDNode {
1825 friend class SelectionDAG;
1826
1827 union {
1828 const Constant *ConstVal;
1829 MachineConstantPoolValue *MachineCPVal;
1830 } Val;
1831 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1832 Align Alignment; // Minimum alignment requirement of CP.
1833 unsigned TargetFlags;
1834
1835 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1836 Align Alignment, unsigned TF)
1837 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1838 DebugLoc(), getSDVTList(VT)),
1839 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1840 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1840, __extension__ __PRETTY_FUNCTION__))
;
1841 Val.ConstVal = c;
1842 }
1843
1844 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1845 Align Alignment, unsigned TF)
1846 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1847 DebugLoc(), getSDVTList(VT)),
1848 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1849 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1849, __extension__ __PRETTY_FUNCTION__))
;
1850 Val.MachineCPVal = v;
1851 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1852 }
1853
1854public:
1855 bool isMachineConstantPoolEntry() const {
1856 return Offset < 0;
1857 }
1858
1859 const Constant *getConstVal() const {
1860 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1860, __extension__ __PRETTY_FUNCTION__))
;
1861 return Val.ConstVal;
1862 }
1863
1864 MachineConstantPoolValue *getMachineCPVal() const {
1865 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1865, __extension__ __PRETTY_FUNCTION__))
;
1866 return Val.MachineCPVal;
1867 }
1868
1869 int getOffset() const {
1870 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1871 }
1872
1873 // Return the alignment of this constant pool object, which is either 0 (for
1874 // default alignment) or the desired value.
1875 Align getAlign() const { return Alignment; }
1876 unsigned getTargetFlags() const { return TargetFlags; }
1877
1878 Type *getType() const;
1879
1880 static bool classof(const SDNode *N) {
1881 return N->getOpcode() == ISD::ConstantPool ||
1882 N->getOpcode() == ISD::TargetConstantPool;
1883 }
1884};
1885
1886/// Completely target-dependent object reference.
1887class TargetIndexSDNode : public SDNode {
1888 friend class SelectionDAG;
1889
1890 unsigned TargetFlags;
1891 int Index;
1892 int64_t Offset;
1893
1894public:
1895 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1896 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1897 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1898
1899 unsigned getTargetFlags() const { return TargetFlags; }
1900 int getIndex() const { return Index; }
1901 int64_t getOffset() const { return Offset; }
1902
1903 static bool classof(const SDNode *N) {
1904 return N->getOpcode() == ISD::TargetIndex;
1905 }
1906};
1907
1908class BasicBlockSDNode : public SDNode {
1909 friend class SelectionDAG;
1910
1911 MachineBasicBlock *MBB;
1912
1913 /// Debug info is meaningful and potentially useful here, but we create
1914 /// blocks out of order when they're jumped to, which makes it a bit
1915 /// harder. Let's see if we need it first.
1916 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1917 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1918 {}
1919
1920public:
1921 MachineBasicBlock *getBasicBlock() const { return MBB; }
1922
1923 static bool classof(const SDNode *N) {
1924 return N->getOpcode() == ISD::BasicBlock;
1925 }
1926};
1927
1928/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1929class BuildVectorSDNode : public SDNode {
1930public:
1931 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1932 explicit BuildVectorSDNode() = delete;
1933
1934 /// Check if this is a constant splat, and if so, find the
1935 /// smallest element size that splats the vector. If MinSplatBits is
1936 /// nonzero, the element size must be at least that large. Note that the
1937 /// splat element may be the entire vector (i.e., a one element vector).
1938 /// Returns the splat element value in SplatValue. Any undefined bits in
1939 /// that value are zero, and the corresponding bits in the SplatUndef mask
1940 /// are set. The SplatBitSize value is set to the splat element size in
1941 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1942 /// undefined. isBigEndian describes the endianness of the target.
1943 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1944 unsigned &SplatBitSize, bool &HasAnyUndefs,
1945 unsigned MinSplatBits = 0,
1946 bool isBigEndian = false) const;
1947
1948 /// Returns the demanded splatted value or a null value if this is not a
1949 /// splat.
1950 ///
1951 /// The DemandedElts mask indicates the elements that must be in the splat.
1952 /// If passed a non-null UndefElements bitvector, it will resize it to match
1953 /// the vector width and set the bits where elements are undef.
1954 SDValue getSplatValue(const APInt &DemandedElts,
1955 BitVector *UndefElements = nullptr) const;
1956
1957 /// Returns the splatted value or a null value if this is not a splat.
1958 ///
1959 /// If passed a non-null UndefElements bitvector, it will resize it to match
1960 /// the vector width and set the bits where elements are undef.
1961 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1962
1963 /// Find the shortest repeating sequence of values in the build vector.
1964 ///
1965 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1966 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1967 ///
1968 /// Currently this must be a power-of-2 build vector.
1969 /// The DemandedElts mask indicates the elements that must be present,
1970 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1971 /// non-null UndefElements bitvector, it will resize it to match the original
1972 /// vector width and set the bits where elements are undef. If result is
1973 /// false, Sequence will be empty.
1974 bool getRepeatedSequence(const APInt &DemandedElts,
1975 SmallVectorImpl<SDValue> &Sequence,
1976 BitVector *UndefElements = nullptr) const;
1977
1978 /// Find the shortest repeating sequence of values in the build vector.
1979 ///
1980 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1981 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1982 ///
1983 /// Currently this must be a power-of-2 build vector.
1984 /// If passed a non-null UndefElements bitvector, it will resize it to match
1985 /// the original vector width and set the bits where elements are undef.
1986 /// If result is false, Sequence will be empty.
1987 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
1988 BitVector *UndefElements = nullptr) const;
1989
1990 /// Returns the demanded splatted constant or null if this is not a constant
1991 /// splat.
1992 ///
1993 /// The DemandedElts mask indicates the elements that must be in the splat.
1994 /// If passed a non-null UndefElements bitvector, it will resize it to match
1995 /// the vector width and set the bits where elements are undef.
1996 ConstantSDNode *
1997 getConstantSplatNode(const APInt &DemandedElts,
1998 BitVector *UndefElements = nullptr) const;
1999
2000 /// Returns the splatted constant or null if this is not a constant
2001 /// splat.
2002 ///
2003 /// If passed a non-null UndefElements bitvector, it will resize it to match
2004 /// the vector width and set the bits where elements are undef.
2005 ConstantSDNode *
2006 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant FP or null if this is not a
2009 /// constant FP splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantFPSDNode *
2015 getConstantFPSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant FP or null if this is not a constant
2019 /// FP splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantFPSDNode *
2024 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// If this is a constant FP splat and the splatted constant FP is an
2027 /// exact power or 2, return the log base 2 integer value. Otherwise,
2028 /// return -1.
2029 ///
2030 /// The BitWidth specifies the necessary bit precision.
2031 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2032 uint32_t BitWidth) const;
2033
2034 bool isConstant() const;
2035
2036 static bool classof(const SDNode *N) {
2037 return N->getOpcode() == ISD::BUILD_VECTOR;
2038 }
2039};
2040
2041/// An SDNode that holds an arbitrary LLVM IR Value. This is
2042/// used when the SelectionDAG needs to make a simple reference to something
2043/// in the LLVM IR representation.
2044///
2045class SrcValueSDNode : public SDNode {
2046 friend class SelectionDAG;
2047
2048 const Value *V;
2049
2050 /// Create a SrcValue for a general value.
2051 explicit SrcValueSDNode(const Value *v)
2052 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2053
2054public:
2055 /// Return the contained Value.
2056 const Value *getValue() const { return V; }
2057
2058 static bool classof(const SDNode *N) {
2059 return N->getOpcode() == ISD::SRCVALUE;
2060 }
2061};
2062
2063class MDNodeSDNode : public SDNode {
2064 friend class SelectionDAG;
2065
2066 const MDNode *MD;
2067
2068 explicit MDNodeSDNode(const MDNode *md)
2069 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2070 {}
2071
2072public:
2073 const MDNode *getMD() const { return MD; }
2074
2075 static bool classof(const SDNode *N) {
2076 return N->getOpcode() == ISD::MDNODE_SDNODE;
2077 }
2078};
2079
2080class RegisterSDNode : public SDNode {
2081 friend class SelectionDAG;
2082
2083 Register Reg;
2084
2085 RegisterSDNode(Register reg, EVT VT)
2086 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2087
2088public:
2089 Register getReg() const { return Reg; }
2090
2091 static bool classof(const SDNode *N) {
2092 return N->getOpcode() == ISD::Register;
2093 }
2094};
2095
2096class RegisterMaskSDNode : public SDNode {
2097 friend class SelectionDAG;
2098
2099 // The memory for RegMask is not owned by the node.
2100 const uint32_t *RegMask;
2101
2102 RegisterMaskSDNode(const uint32_t *mask)
2103 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2104 RegMask(mask) {}
2105
2106public:
2107 const uint32_t *getRegMask() const { return RegMask; }
2108
2109 static bool classof(const SDNode *N) {
2110 return N->getOpcode() == ISD::RegisterMask;
2111 }
2112};
2113
2114class BlockAddressSDNode : public SDNode {
2115 friend class SelectionDAG;
2116
2117 const BlockAddress *BA;
2118 int64_t Offset;
2119 unsigned TargetFlags;
2120
2121 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2122 int64_t o, unsigned Flags)
2123 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2124 BA(ba), Offset(o), TargetFlags(Flags) {}
2125
2126public:
2127 const BlockAddress *getBlockAddress() const { return BA; }
2128 int64_t getOffset() const { return Offset; }
2129 unsigned getTargetFlags() const { return TargetFlags; }
2130
2131 static bool classof(const SDNode *N) {
2132 return N->getOpcode() == ISD::BlockAddress ||
2133 N->getOpcode() == ISD::TargetBlockAddress;
2134 }
2135};
2136
2137class LabelSDNode : public SDNode {
2138 friend class SelectionDAG;
2139
2140 MCSymbol *Label;
2141
2142 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2143 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2144 assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
"not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2144, __extension__ __PRETTY_FUNCTION__))
;
2145 }
2146
2147public:
2148 MCSymbol *getLabel() const { return Label; }
2149
2150 static bool classof(const SDNode *N) {
2151 return N->getOpcode() == ISD::EH_LABEL ||
2152 N->getOpcode() == ISD::ANNOTATION_LABEL;
2153 }
2154};
2155
2156class ExternalSymbolSDNode : public SDNode {
2157 friend class SelectionDAG;
2158
2159 const char *Symbol;
2160 unsigned TargetFlags;
2161
2162 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2163 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2164 DebugLoc(), getSDVTList(VT)),
2165 Symbol(Sym), TargetFlags(TF) {}
2166
2167public:
2168 const char *getSymbol() const { return Symbol; }
2169 unsigned getTargetFlags() const { return TargetFlags; }
2170
2171 static bool classof(const SDNode *N) {
2172 return N->getOpcode() == ISD::ExternalSymbol ||
2173 N->getOpcode() == ISD::TargetExternalSymbol;
2174 }
2175};
2176
2177class MCSymbolSDNode : public SDNode {
2178 friend class SelectionDAG;
2179
2180 MCSymbol *Symbol;
2181
2182 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2183 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2184
2185public:
2186 MCSymbol *getMCSymbol() const { return Symbol; }
2187
2188 static bool classof(const SDNode *N) {
2189 return N->getOpcode() == ISD::MCSymbol;
2190 }
2191};
2192
2193class CondCodeSDNode : public SDNode {
2194 friend class SelectionDAG;
2195
2196 ISD::CondCode Condition;
2197
2198 explicit CondCodeSDNode(ISD::CondCode Cond)
2199 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2200 Condition(Cond) {}
2201
2202public:
2203 ISD::CondCode get() const { return Condition; }
2204
2205 static bool classof(const SDNode *N) {
2206 return N->getOpcode() == ISD::CONDCODE;
2207 }
2208};
2209
2210/// This class is used to represent EVT's, which are used
2211/// to parameterize some operations.
2212class VTSDNode : public SDNode {
2213 friend class SelectionDAG;
2214
2215 EVT ValueType;
2216
2217 explicit VTSDNode(EVT VT)
2218 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2219 ValueType(VT) {}
2220
2221public:
2222 EVT getVT() const { return ValueType; }
2223
2224 static bool classof(const SDNode *N) {
2225 return N->getOpcode() == ISD::VALUETYPE;
2226 }
2227};
2228
2229/// Base class for LoadSDNode and StoreSDNode
2230class LSBaseSDNode : public MemSDNode {
2231public:
2232 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2233 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2234 MachineMemOperand *MMO)
2235 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2236 LSBaseSDNodeBits.AddressingMode = AM;
2237 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2237, __extension__ __PRETTY_FUNCTION__))
;
2238 }
2239
2240 const SDValue &getOffset() const {
2241 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2242 }
2243
2244 /// Return the addressing mode for this load or store:
2245 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2246 ISD::MemIndexedMode getAddressingMode() const {
2247 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2248 }
2249
2250 /// Return true if this is a pre/post inc/dec load/store.
2251 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2252
2253 /// Return true if this is NOT a pre/post inc/dec load/store.
2254 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2255
2256 static bool classof(const SDNode *N) {
2257 return N->getOpcode() == ISD::LOAD ||
2258 N->getOpcode() == ISD::STORE;
2259 }
2260};
2261
2262/// This class is used to represent ISD::LOAD nodes.
2263class LoadSDNode : public LSBaseSDNode {
2264 friend class SelectionDAG;
2265
2266 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2267 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2268 MachineMemOperand *MMO)
2269 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2270 LoadSDNodeBits.ExtTy = ETy;
2271 assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2271, __extension__ __PRETTY_FUNCTION__))
;
2272 assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2272, __extension__ __PRETTY_FUNCTION__))
;
2273 }
2274
2275public:
2276 /// Return whether this is a plain node,
2277 /// or one of the varieties of value-extending loads.
2278 ISD::LoadExtType getExtensionType() const {
2279 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2280 }
2281
2282 const SDValue &getBasePtr() const { return getOperand(1); }
2283 const SDValue &getOffset() const { return getOperand(2); }
2284
2285 static bool classof(const SDNode *N) {
2286 return N->getOpcode() == ISD::LOAD;
2287 }
2288};
2289
2290/// This class is used to represent ISD::STORE nodes.
2291class StoreSDNode : public LSBaseSDNode {
2292 friend class SelectionDAG;
2293
2294 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2295 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2296 MachineMemOperand *MMO)
2297 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2298 StoreSDNodeBits.IsTruncating = isTrunc;
2299 assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2299, __extension__ __PRETTY_FUNCTION__))
;
2300 assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2300, __extension__ __PRETTY_FUNCTION__))
;
2301 }
2302
2303public:
2304 /// Return true if the op does a truncation before store.
2305 /// For integers this is the same as doing a TRUNCATE and storing the result.
2306 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2307 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2308 void setTruncatingStore(bool Truncating) {
2309 StoreSDNodeBits.IsTruncating = Truncating;
2310 }
2311
2312 const SDValue &getValue() const { return getOperand(1); }
2313 const SDValue &getBasePtr() const { return getOperand(2); }
2314 const SDValue &getOffset() const { return getOperand(3); }
2315
2316 static bool classof(const SDNode *N) {
2317 return N->getOpcode() == ISD::STORE;
2318 }
2319};
2320
2321/// This base class is used to represent MLOAD and MSTORE nodes
2322class MaskedLoadStoreSDNode : public MemSDNode {
2323public:
2324 friend class SelectionDAG;
2325
2326 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2327 const DebugLoc &dl, SDVTList VTs,
2328 ISD::MemIndexedMode AM, EVT MemVT,
2329 MachineMemOperand *MMO)
2330 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2331 LSBaseSDNodeBits.AddressingMode = AM;
2332 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2332, __extension__ __PRETTY_FUNCTION__))
;
2333 }
2334
2335 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2336 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2337 // Mask is a vector of i1 elements
2338 const SDValue &getOffset() const {
2339 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2340 }
2341 const SDValue &getMask() const {
2342 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2343 }
2344
2345 /// Return the addressing mode for this load or store:
2346 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2347 ISD::MemIndexedMode getAddressingMode() const {
2348 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2349 }
2350
2351 /// Return true if this is a pre/post inc/dec load/store.
2352 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2353
2354 /// Return true if this is NOT a pre/post inc/dec load/store.
2355 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2356
2357 static bool classof(const SDNode *N) {
2358 return N->getOpcode() == ISD::MLOAD ||
2359 N->getOpcode() == ISD::MSTORE;
2360 }
2361};
2362
2363/// This class is used to represent an MLOAD node
2364class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2365public:
2366 friend class SelectionDAG;
2367
2368 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2369 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2370 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2371 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2372 LoadSDNodeBits.ExtTy = ETy;
2373 LoadSDNodeBits.IsExpanding = IsExpanding;
2374 }
2375
2376 ISD::LoadExtType getExtensionType() const {
2377 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2378 }
2379
2380 const SDValue &getBasePtr() const { return getOperand(1); }
2381 const SDValue &getOffset() const { return getOperand(2); }
2382 const SDValue &getMask() const { return getOperand(3); }
2383 const SDValue &getPassThru() const { return getOperand(4); }
2384
2385 static bool classof(const SDNode *N) {
2386 return N->getOpcode() == ISD::MLOAD;
2387 }
2388
2389 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2390};
2391
2392/// This class is used to represent an MSTORE node
2393class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2394public:
2395 friend class SelectionDAG;
2396
2397 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2398 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2399 EVT MemVT, MachineMemOperand *MMO)
2400 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2401 StoreSDNodeBits.IsTruncating = isTrunc;
2402 StoreSDNodeBits.IsCompressing = isCompressing;
2403 }
2404
2405 /// Return true if the op does a truncation before store.
2406 /// For integers this is the same as doing a TRUNCATE and storing the result.
2407 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2408 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2409
2410 /// Returns true if the op does a compression to the vector before storing.
2411 /// The node contiguously stores the active elements (integers or floats)
2412 /// in src (those with their respective bit set in writemask k) to unaligned
2413 /// memory at base_addr.
2414 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2415
2416 const SDValue &getValue() const { return getOperand(1); }
2417 const SDValue &getBasePtr() const { return getOperand(2); }
2418 const SDValue &getOffset() const { return getOperand(3); }
2419 const SDValue &getMask() const { return getOperand(4); }
2420
2421 static bool classof(const SDNode *N) {
2422 return N->getOpcode() == ISD::MSTORE;
2423 }
2424};
2425
2426/// This is a base class used to represent
2427/// MGATHER and MSCATTER nodes
2428///
2429class MaskedGatherScatterSDNode : public MemSDNode {
2430public:
2431 friend class SelectionDAG;
2432
2433 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2434 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2435 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2436 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2437 LSBaseSDNodeBits.AddressingMode = IndexType;
2438 assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
"Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2438, __extension__ __PRETTY_FUNCTION__))
;
2439 }
2440
2441 /// How is Index applied to BasePtr when computing addresses.
2442 ISD::MemIndexType getIndexType() const {
2443 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2444 }
2445 void setIndexType(ISD::MemIndexType IndexType) {
2446 LSBaseSDNodeBits.AddressingMode = IndexType;
2447 }
2448 bool isIndexScaled() const {
2449 return (getIndexType() == ISD::SIGNED_SCALED) ||
2450 (getIndexType() == ISD::UNSIGNED_SCALED);
2451 }
2452 bool isIndexSigned() const {
2453 return (getIndexType() == ISD::SIGNED_SCALED) ||
2454 (getIndexType() == ISD::SIGNED_UNSCALED);
2455 }
2456
2457 // In the both nodes address is Op1, mask is Op2:
2458 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2459 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2460 // Mask is a vector of i1 elements
2461 const SDValue &getBasePtr() const { return getOperand(3); }
2462 const SDValue &getIndex() const { return getOperand(4); }
2463 const SDValue &getMask() const { return getOperand(2); }
2464 const SDValue &getScale() const { return getOperand(5); }
2465
2466 static bool classof(const SDNode *N) {
2467 return N->getOpcode() == ISD::MGATHER ||
2468 N->getOpcode() == ISD::MSCATTER;
2469 }
2470};
2471
2472/// This class is used to represent an MGATHER node
2473///
2474class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2475public:
2476 friend class SelectionDAG;
2477
2478 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2479 EVT MemVT, MachineMemOperand *MMO,
2480 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2481 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2482 IndexType) {
2483 LoadSDNodeBits.ExtTy = ETy;
2484 }
2485
2486 const SDValue &getPassThru() const { return getOperand(1); }
2487
2488 ISD::LoadExtType getExtensionType() const {
2489 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2490 }
2491
2492 static bool classof(const SDNode *N) {
2493 return N->getOpcode() == ISD::MGATHER;
2494 }
2495};
2496
2497/// This class is used to represent an MSCATTER node
2498///
2499class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2500public:
2501 friend class SelectionDAG;
2502
2503 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2504 EVT MemVT, MachineMemOperand *MMO,
2505 ISD::MemIndexType IndexType, bool IsTrunc)
2506 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2507 IndexType) {
2508 StoreSDNodeBits.IsTruncating = IsTrunc;
2509 }
2510
2511 /// Return true if the op does a truncation before store.
2512 /// For integers this is the same as doing a TRUNCATE and storing the result.
2513 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2514 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2515
2516 const SDValue &getValue() const { return getOperand(1); }
2517
2518 static bool classof(const SDNode *N) {
2519 return N->getOpcode() == ISD::MSCATTER;
2520 }
2521};
2522
2523/// An SDNode that represents everything that will be needed
2524/// to construct a MachineInstr. These nodes are created during the
2525/// instruction selection proper phase.
2526///
2527/// Note that the only supported way to set the `memoperands` is by calling the
2528/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2529/// inside the DAG rather than in the node.
2530class MachineSDNode : public SDNode {
2531private:
2532 friend class SelectionDAG;
2533
2534 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2535 : SDNode(Opc, Order, DL, VTs) {}
2536
2537 // We use a pointer union between a single `MachineMemOperand` pointer and
2538 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2539 // the number of these is zero, the single pointer variant used when the
2540 // number is one, and the array is used for larger numbers.
2541 //
2542 // The array is allocated via the `SelectionDAG`'s allocator and so will
2543 // always live until the DAG is cleaned up and doesn't require ownership here.
2544 //
2545 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2546 // subclasses aren't managed in a conforming C++ manner. See the comments on
2547 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2548 // constraint here is that these don't manage memory with their constructor or
2549 // destructor and can be initialized to a good state even if they start off
2550 // uninitialized.
2551 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2552
2553 // Note that this could be folded into the above `MemRefs` member if doing so
2554 // is advantageous at some point. We don't need to store this in most cases.
2555 // However, at the moment this doesn't appear to make the allocation any
2556 // smaller and makes the code somewhat simpler to read.
2557 int NumMemRefs = 0;
2558
2559public:
2560 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2561
2562 ArrayRef<MachineMemOperand *> memoperands() const {
2563 // Special case the common cases.
2564 if (NumMemRefs == 0)
2565 return {};
2566 if (NumMemRefs == 1)
2567 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2568
2569 // Otherwise we have an actual array.
2570 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2571 }
2572 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2573 mmo_iterator memoperands_end() const { return memoperands().end(); }
2574 bool memoperands_empty() const { return memoperands().empty(); }
2575
2576 /// Clear out the memory reference descriptor list.
2577 void clearMemRefs() {
2578 MemRefs = nullptr;
2579 NumMemRefs = 0;
2580 }
2581
2582 static bool classof(const SDNode *N) {
2583 return N->isMachineOpcode();
2584 }
2585};
2586
2587/// An SDNode that records if a register contains a value that is guaranteed to
2588/// be aligned accordingly.
2589class AssertAlignSDNode : public SDNode {
2590 Align Alignment;
2591
2592public:
2593 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2594 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2595
2596 Align getAlign() const { return Alignment; }
2597
2598 static bool classof(const SDNode *N) {
2599 return N->getOpcode() == ISD::AssertAlign;
2600 }
2601};
2602
2603class SDNodeIterator {
2604 const SDNode *Node;
2605 unsigned Operand;
2606
2607 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2608
2609public:
2610 using iterator_category = std::forward_iterator_tag;
2611 using value_type = SDNode;
2612 using difference_type = std::ptrdiff_t;
2613 using pointer = value_type *;
2614 using reference = value_type &;
2615
2616 bool operator==(const SDNodeIterator& x) const {
2617 return Operand == x.Operand;
2618 }
2619 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2620
2621 pointer operator*() const {
2622 return Node->getOperand(Operand).getNode();
2623 }
2624 pointer operator->() const { return operator*(); }
2625
2626 SDNodeIterator& operator++() { // Preincrement
2627 ++Operand;
2628 return *this;
2629 }
2630 SDNodeIterator operator++(int) { // Postincrement
2631 SDNodeIterator tmp = *this; ++*this; return tmp;
2632 }
2633 size_t operator-(SDNodeIterator Other) const {
2634 assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2635, __extension__ __PRETTY_FUNCTION__))
2635 "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2635, __extension__ __PRETTY_FUNCTION__))
;
2636 return Operand - Other.Operand;
2637 }
2638
2639 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2640 static SDNodeIterator end (const SDNode *N) {
2641 return SDNodeIterator(N, N->getNumOperands());
2642 }
2643
2644 unsigned getOperand() const { return Operand; }
2645 const SDNode *getNode() const { return Node; }
2646};
2647
2648template <> struct GraphTraits<SDNode*> {
2649 using NodeRef = SDNode *;
2650 using ChildIteratorType = SDNodeIterator;
2651
2652 static NodeRef getEntryNode(SDNode *N) { return N; }
2653
2654 static ChildIteratorType child_begin(NodeRef N) {
2655 return SDNodeIterator::begin(N);
2656 }
2657
2658 static ChildIteratorType child_end(NodeRef N) {
2659 return SDNodeIterator::end(N);
2660 }
2661};
2662
2663/// A representation of the largest SDNode, for use in sizeof().
2664///
2665/// This needs to be a union because the largest node differs on 32 bit systems
2666/// with 4 and 8 byte pointer alignment, respectively.
2667using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
2668 BlockAddressSDNode,
2669 GlobalAddressSDNode,
2670 PseudoProbeSDNode>;
2671
2672/// The SDNode class with the greatest alignment requirement.
2673using MostAlignedSDNode = GlobalAddressSDNode;
2674
2675namespace ISD {
2676
2677 /// Returns true if the specified node is a non-extending and unindexed load.
2678 inline bool isNormalLoad(const SDNode *N) {
2679 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
2680 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
2681 Ld->getAddressingMode() == ISD::UNINDEXED;
2682 }
2683
2684 /// Returns true if the specified node is a non-extending load.
2685 inline bool isNON_EXTLoad(const SDNode *N) {
2686 return isa<LoadSDNode>(N) &&
2687 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
2688 }
2689
2690 /// Returns true if the specified node is a EXTLOAD.
2691 inline bool isEXTLoad(const SDNode *N) {
2692 return isa<LoadSDNode>(N) &&
2693 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
2694 }
2695
2696 /// Returns true if the specified node is a SEXTLOAD.
2697 inline bool isSEXTLoad(const SDNode *N) {
2698 return isa<LoadSDNode>(N) &&
2699 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
2700 }
2701
2702 /// Returns true if the specified node is a ZEXTLOAD.
2703 inline bool isZEXTLoad(const SDNode *N) {
2704 return isa<LoadSDNode>(N) &&
2705 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
2706 }
2707
2708 /// Returns true if the specified node is an unindexed load.
2709 inline bool isUNINDEXEDLoad(const SDNode *N) {
2710 return isa<LoadSDNode>(N) &&
2711 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2712 }
2713
2714 /// Returns true if the specified node is a non-truncating
2715 /// and unindexed store.
2716 inline bool isNormalStore(const SDNode *N) {
2717 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
2718 return St && !St->isTruncatingStore() &&
2719 St->getAddressingMode() == ISD::UNINDEXED;
2720 }
2721
2722 /// Returns true if the specified node is an unindexed store.
2723 inline bool isUNINDEXEDStore(const SDNode *N) {
2724 return isa<StoreSDNode>(N) &&
2725 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2726 }
2727
2728 /// Attempt to match a unary predicate against a scalar/splat constant or
2729 /// every element of a constant BUILD_VECTOR.
2730 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2731 bool matchUnaryPredicate(SDValue Op,
2732 std::function<bool(ConstantSDNode *)> Match,
2733 bool AllowUndefs = false);
2734
2735 /// Attempt to match a binary predicate against a pair of scalar/splat
2736 /// constants or every element of a pair of constant BUILD_VECTORs.
2737 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2738 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
2739 bool matchBinaryPredicate(
2740 SDValue LHS, SDValue RHS,
2741 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
2742 bool AllowUndefs = false, bool AllowTypeMismatch = false);
2743
2744 /// Returns true if the specified value is the overflow result from one
2745 /// of the overflow intrinsic nodes.
2746 inline bool isOverflowIntrOpRes(SDValue Op) {
2747 unsigned Opc = Op.getOpcode();
2748 return (Op.getResNo() == 1 &&
2749 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2750 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2751 }
2752
2753} // end namespace ISD
2754
2755} // end namespace llvm
2756
2757#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H