LLVM 19.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
96#include "llvm/Support/Debug.h"
105#include <cstddef>
106#include <iterator>
107#include <limits>
108#include <optional>
109#include <tuple>
110
111using namespace llvm;
112using namespace PatternMatch;
113using namespace SwitchCG;
114
115#define DEBUG_TYPE "isel"
116
117/// LimitFloatPrecision - Generate low-precision inline sequences for
118/// some float libcalls (6, 8 or 12 bits).
119static unsigned LimitFloatPrecision;
120
121static cl::opt<bool>
122 InsertAssertAlign("insert-assert-align", cl::init(true),
123 cl::desc("Insert the experimental `assertalign` node."),
125
127 LimitFPPrecision("limit-float-precision",
128 cl::desc("Generate low-precision inline sequences "
129 "for some float libcalls"),
131 cl::init(0));
132
134 "switch-peel-threshold", cl::Hidden, cl::init(66),
135 cl::desc("Set the case probability threshold for peeling the case from a "
136 "switch statement. A value greater than 100 will void this "
137 "optimization"));
138
139// Limit the width of DAG chains. This is important in general to prevent
140// DAG-based analysis from blowing up. For example, alias analysis and
141// load clustering may not complete in reasonable time. It is difficult to
142// recognize and avoid this situation within each individual analysis, and
143// future analyses are likely to have the same behavior. Limiting DAG width is
144// the safe approach and will be especially important with global DAGs.
145//
146// MaxParallelChains default is arbitrarily high to avoid affecting
147// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
148// sequence over this should have been converted to llvm.memcpy by the
149// frontend. It is easy to induce this behavior with .ll code such as:
150// %buffer = alloca [4096 x i8]
151// %data = load [4096 x i8]* %argPtr
152// store [4096 x i8] %data, [4096 x i8]* %buffer
153static const unsigned MaxParallelChains = 64;
154
156 const SDValue *Parts, unsigned NumParts,
157 MVT PartVT, EVT ValueVT, const Value *V,
158 SDValue InChain,
159 std::optional<CallingConv::ID> CC);
160
161/// getCopyFromParts - Create a value that contains the specified legal parts
162/// combined into the value they represent. If the parts combine to a type
163/// larger than ValueVT then AssertOp can be used to specify whether the extra
164/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
165/// (ISD::AssertSext).
166static SDValue
167getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
168 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
169 SDValue InChain,
170 std::optional<CallingConv::ID> CC = std::nullopt,
171 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
172 // Let the target assemble the parts if it wants to
173 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
174 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
175 PartVT, ValueVT, CC))
176 return Val;
177
178 if (ValueVT.isVector())
179 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
180 InChain, CC);
181
182 assert(NumParts > 0 && "No parts to assemble!");
183 SDValue Val = Parts[0];
184
185 if (NumParts > 1) {
186 // Assemble the value from multiple parts.
187 if (ValueVT.isInteger()) {
188 unsigned PartBits = PartVT.getSizeInBits();
189 unsigned ValueBits = ValueVT.getSizeInBits();
190
191 // Assemble the power of 2 part.
192 unsigned RoundParts = llvm::bit_floor(NumParts);
193 unsigned RoundBits = PartBits * RoundParts;
194 EVT RoundVT = RoundBits == ValueBits ?
195 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
196 SDValue Lo, Hi;
197
198 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
199
200 if (RoundParts > 2) {
201 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
202 InChain);
203 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
204 PartVT, HalfVT, V, InChain);
205 } else {
206 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
207 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
208 }
209
210 if (DAG.getDataLayout().isBigEndian())
211 std::swap(Lo, Hi);
212
213 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
214
215 if (RoundParts < NumParts) {
216 // Assemble the trailing non-power-of-2 part.
217 unsigned OddParts = NumParts - RoundParts;
218 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
219 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
220 OddVT, V, InChain, CC);
221
222 // Combine the round and odd parts.
223 Lo = Val;
224 if (DAG.getDataLayout().isBigEndian())
225 std::swap(Lo, Hi);
226 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
227 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
228 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
229 DAG.getConstant(Lo.getValueSizeInBits(), DL,
231 TotalVT, DAG.getDataLayout())));
232 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
233 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
234 }
235 } else if (PartVT.isFloatingPoint()) {
236 // FP split into multiple FP parts (for ppcf128)
237 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
238 "Unexpected split");
239 SDValue Lo, Hi;
240 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
241 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
242 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
243 std::swap(Lo, Hi);
244 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
245 } else {
246 // FP split into integer parts (soft fp)
247 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
248 !PartVT.isVector() && "Unexpected split");
249 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
250 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
251 InChain, CC);
252 }
253 }
254
255 // There is now one part, held in Val. Correct it to match ValueVT.
256 // PartEVT is the type of the register class that holds the value.
257 // ValueVT is the type of the inline asm operation.
258 EVT PartEVT = Val.getValueType();
259
260 if (PartEVT == ValueVT)
261 return Val;
262
263 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
264 ValueVT.bitsLT(PartEVT)) {
265 // For an FP value in an integer part, we need to truncate to the right
266 // width first.
267 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
268 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
269 }
270
271 // Handle types that have the same size.
272 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
273 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
274
275 // Handle types with different sizes.
276 if (PartEVT.isInteger() && ValueVT.isInteger()) {
277 if (ValueVT.bitsLT(PartEVT)) {
278 // For a truncate, see if we have any information to
279 // indicate whether the truncated bits will always be
280 // zero or sign-extension.
281 if (AssertOp)
282 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
283 DAG.getValueType(ValueVT));
284 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
285 }
286 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
287 }
288
289 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
290 // FP_ROUND's are always exact here.
291 if (ValueVT.bitsLT(Val.getValueType())) {
292
293 SDValue NoChange =
295
297 llvm::Attribute::StrictFP)) {
298 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
299 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
300 NoChange);
301 }
302
303 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
304 }
305
306 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
307 }
308
309 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
310 // then truncating.
311 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
312 ValueVT.bitsLT(PartEVT)) {
313 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
314 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
315 }
316
317 report_fatal_error("Unknown mismatch in getCopyFromParts!");
318}
319
321 const Twine &ErrMsg) {
322 const Instruction *I = dyn_cast_or_null<Instruction>(V);
323 if (!V)
324 return Ctx.emitError(ErrMsg);
325
326 const char *AsmError = ", possible invalid constraint for vector type";
327 if (const CallInst *CI = dyn_cast<CallInst>(I))
328 if (CI->isInlineAsm())
329 return Ctx.emitError(I, ErrMsg + AsmError);
330
331 return Ctx.emitError(I, ErrMsg);
332}
333
334/// getCopyFromPartsVector - Create a value that contains the specified legal
335/// parts combined into the value they represent. If the parts combine to a
336/// type larger than ValueVT then AssertOp can be used to specify whether the
337/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
338/// ValueVT (ISD::AssertSext).
340 const SDValue *Parts, unsigned NumParts,
341 MVT PartVT, EVT ValueVT, const Value *V,
342 SDValue InChain,
343 std::optional<CallingConv::ID> CallConv) {
344 assert(ValueVT.isVector() && "Not a vector value");
345 assert(NumParts > 0 && "No parts to assemble!");
346 const bool IsABIRegCopy = CallConv.has_value();
347
348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
349 SDValue Val = Parts[0];
350
351 // Handle a multi-element vector.
352 if (NumParts > 1) {
353 EVT IntermediateVT;
354 MVT RegisterVT;
355 unsigned NumIntermediates;
356 unsigned NumRegs;
357
358 if (IsABIRegCopy) {
360 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
361 NumIntermediates, RegisterVT);
362 } else {
363 NumRegs =
364 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
365 NumIntermediates, RegisterVT);
366 }
367
368 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
369 NumParts = NumRegs; // Silence a compiler warning.
370 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
371 assert(RegisterVT.getSizeInBits() ==
372 Parts[0].getSimpleValueType().getSizeInBits() &&
373 "Part type sizes don't match!");
374
375 // Assemble the parts into intermediate operands.
376 SmallVector<SDValue, 8> Ops(NumIntermediates);
377 if (NumIntermediates == NumParts) {
378 // If the register was not expanded, truncate or copy the value,
379 // as appropriate.
380 for (unsigned i = 0; i != NumParts; ++i)
381 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
382 V, InChain, CallConv);
383 } else if (NumParts > 0) {
384 // If the intermediate type was expanded, build the intermediate
385 // operands from the parts.
386 assert(NumParts % NumIntermediates == 0 &&
387 "Must expand into a divisible number of parts!");
388 unsigned Factor = NumParts / NumIntermediates;
389 for (unsigned i = 0; i != NumIntermediates; ++i)
390 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
391 IntermediateVT, V, InChain, CallConv);
392 }
393
394 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
395 // intermediate operands.
396 EVT BuiltVectorTy =
397 IntermediateVT.isVector()
399 *DAG.getContext(), IntermediateVT.getScalarType(),
400 IntermediateVT.getVectorElementCount() * NumParts)
402 IntermediateVT.getScalarType(),
403 NumIntermediates);
404 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
406 DL, BuiltVectorTy, Ops);
407 }
408
409 // There is now one part, held in Val. Correct it to match ValueVT.
410 EVT PartEVT = Val.getValueType();
411
412 if (PartEVT == ValueVT)
413 return Val;
414
415 if (PartEVT.isVector()) {
416 // Vector/Vector bitcast.
417 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
418 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
419
420 // If the parts vector has more elements than the value vector, then we
421 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
422 // Extract the elements we want.
423 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
426 (PartEVT.getVectorElementCount().isScalable() ==
427 ValueVT.getVectorElementCount().isScalable()) &&
428 "Cannot narrow, it would be a lossy transformation");
429 PartEVT =
431 ValueVT.getVectorElementCount());
432 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
433 DAG.getVectorIdxConstant(0, DL));
434 if (PartEVT == ValueVT)
435 return Val;
436 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
437 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
438
439 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
440 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
441 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
442 }
443
444 // Promoted vector extract
445 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
446 }
447
448 // Trivial bitcast if the types are the same size and the destination
449 // vector type is legal.
450 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
451 TLI.isTypeLegal(ValueVT))
452 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
453
454 if (ValueVT.getVectorNumElements() != 1) {
455 // Certain ABIs require that vectors are passed as integers. For vectors
456 // are the same size, this is an obvious bitcast.
457 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
458 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
459 } else if (ValueVT.bitsLT(PartEVT)) {
460 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
461 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
462 // Drop the extra bits.
463 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
464 return DAG.getBitcast(ValueVT, Val);
465 }
466
468 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
469 return DAG.getUNDEF(ValueVT);
470 }
471
472 // Handle cases such as i8 -> <1 x i1>
473 EVT ValueSVT = ValueVT.getVectorElementType();
474 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
475 unsigned ValueSize = ValueSVT.getSizeInBits();
476 if (ValueSize == PartEVT.getSizeInBits()) {
477 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
478 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
479 // It's possible a scalar floating point type gets softened to integer and
480 // then promoted to a larger integer. If PartEVT is the larger integer
481 // we need to truncate it and then bitcast to the FP type.
482 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
483 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
484 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
485 Val = DAG.getBitcast(ValueSVT, Val);
486 } else {
487 Val = ValueVT.isFloatingPoint()
488 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
489 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
490 }
491 }
492
493 return DAG.getBuildVector(ValueVT, DL, Val);
494}
495
496static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
497 SDValue Val, SDValue *Parts, unsigned NumParts,
498 MVT PartVT, const Value *V,
499 std::optional<CallingConv::ID> CallConv);
500
501/// getCopyToParts - Create a series of nodes that contain the specified value
502/// split into legal parts. If the parts contain more bits than Val, then, for
503/// integers, ExtendKind can be used to specify how to generate the extra bits.
504static void
506 unsigned NumParts, MVT PartVT, const Value *V,
507 std::optional<CallingConv::ID> CallConv = std::nullopt,
508 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
509 // Let the target split the parts if it wants to
510 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
511 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
512 CallConv))
513 return;
514 EVT ValueVT = Val.getValueType();
515
516 // Handle the vector case separately.
517 if (ValueVT.isVector())
518 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
519 CallConv);
520
521 unsigned OrigNumParts = NumParts;
523 "Copying to an illegal type!");
524
525 if (NumParts == 0)
526 return;
527
528 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
529 EVT PartEVT = PartVT;
530 if (PartEVT == ValueVT) {
531 assert(NumParts == 1 && "No-op copy with multiple parts!");
532 Parts[0] = Val;
533 return;
534 }
535
536 unsigned PartBits = PartVT.getSizeInBits();
537 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
538 // If the parts cover more bits than the value has, promote the value.
539 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
540 assert(NumParts == 1 && "Do not know what to promote to!");
541 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
542 } else {
543 if (ValueVT.isFloatingPoint()) {
544 // FP values need to be bitcast, then extended if they are being put
545 // into a larger container.
546 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
547 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
548 }
549 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
550 ValueVT.isInteger() &&
551 "Unknown mismatch!");
552 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
553 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
554 if (PartVT == MVT::x86mmx)
555 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
556 }
557 } else if (PartBits == ValueVT.getSizeInBits()) {
558 // Different types of the same size.
559 assert(NumParts == 1 && PartEVT != ValueVT);
560 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
562 // If the parts cover less bits than value has, truncate the value.
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571
572 // The value may have changed - recompute ValueVT.
573 ValueVT = Val.getValueType();
574 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
575 "Failed to tile the value with PartVT!");
576
577 if (NumParts == 1) {
578 if (PartEVT != ValueVT) {
580 "scalar-to-vector conversion failed");
581 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
582 }
583
584 Parts[0] = Val;
585 return;
586 }
587
588 // Expand the value into multiple parts.
589 if (NumParts & (NumParts - 1)) {
590 // The number of parts is not a power of 2. Split off and copy the tail.
591 assert(PartVT.isInteger() && ValueVT.isInteger() &&
592 "Do not know what to expand to!");
593 unsigned RoundParts = llvm::bit_floor(NumParts);
594 unsigned RoundBits = RoundParts * PartBits;
595 unsigned OddParts = NumParts - RoundParts;
596 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
597 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
598
599 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
600 CallConv);
601
602 if (DAG.getDataLayout().isBigEndian())
603 // The odd parts were reversed by getCopyToParts - unreverse them.
604 std::reverse(Parts + RoundParts, Parts + NumParts);
605
606 NumParts = RoundParts;
607 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
608 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
609 }
610
611 // The number of parts is a power of 2. Repeatedly bisect the value using
612 // EXTRACT_ELEMENT.
613 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
615 ValueVT.getSizeInBits()),
616 Val);
617
618 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
619 for (unsigned i = 0; i < NumParts; i += StepSize) {
620 unsigned ThisBits = StepSize * PartBits / 2;
621 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
622 SDValue &Part0 = Parts[i];
623 SDValue &Part1 = Parts[i+StepSize/2];
624
625 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
626 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
627 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
628 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
629
630 if (ThisBits == PartBits && ThisVT != PartVT) {
631 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
632 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
633 }
634 }
635 }
636
637 if (DAG.getDataLayout().isBigEndian())
638 std::reverse(Parts, Parts + OrigNumParts);
639}
640
642 const SDLoc &DL, EVT PartVT) {
643 if (!PartVT.isVector())
644 return SDValue();
645
646 EVT ValueVT = Val.getValueType();
647 EVT PartEVT = PartVT.getVectorElementType();
648 EVT ValueEVT = ValueVT.getVectorElementType();
649 ElementCount PartNumElts = PartVT.getVectorElementCount();
650 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
651
652 // We only support widening vectors with equivalent element types and
653 // fixed/scalable properties. If a target needs to widen a fixed-length type
654 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
655 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
656 PartNumElts.isScalable() != ValueNumElts.isScalable())
657 return SDValue();
658
659 // Have a try for bf16 because some targets share its ABI with fp16.
660 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
662 "Cannot widen to illegal type");
663 Val = DAG.getNode(ISD::BITCAST, DL,
664 ValueVT.changeVectorElementType(MVT::f16), Val);
665 } else if (PartEVT != ValueEVT) {
666 return SDValue();
667 }
668
669 // Widening a scalable vector to another scalable vector is done by inserting
670 // the vector into a larger undef one.
671 if (PartNumElts.isScalable())
672 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
673 Val, DAG.getVectorIdxConstant(0, DL));
674
675 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
676 // undef elements.
678 DAG.ExtractVectorElements(Val, Ops);
679 SDValue EltUndef = DAG.getUNDEF(PartEVT);
680 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
681
682 // FIXME: Use CONCAT for 2x -> 4x.
683 return DAG.getBuildVector(PartVT, DL, Ops);
684}
685
686/// getCopyToPartsVector - Create a series of nodes that contain the specified
687/// value split into legal parts.
688static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
689 SDValue Val, SDValue *Parts, unsigned NumParts,
690 MVT PartVT, const Value *V,
691 std::optional<CallingConv::ID> CallConv) {
692 EVT ValueVT = Val.getValueType();
693 assert(ValueVT.isVector() && "Not a vector");
694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
695 const bool IsABIRegCopy = CallConv.has_value();
696
697 if (NumParts == 1) {
698 EVT PartEVT = PartVT;
699 if (PartEVT == ValueVT) {
700 // Nothing to do.
701 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
702 // Bitconvert vector->vector case.
703 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
704 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
705 Val = Widened;
706 } else if (PartVT.isVector() &&
708 ValueVT.getVectorElementType()) &&
709 PartEVT.getVectorElementCount() ==
710 ValueVT.getVectorElementCount()) {
711
712 // Promoted vector extract
713 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
714 } else if (PartEVT.isVector() &&
715 PartEVT.getVectorElementType() !=
716 ValueVT.getVectorElementType() &&
717 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
718 TargetLowering::TypeWidenVector) {
719 // Combination of widening and promotion.
720 EVT WidenVT =
722 PartVT.getVectorElementCount());
723 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
724 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
725 } else {
726 // Don't extract an integer from a float vector. This can happen if the
727 // FP type gets softened to integer and then promoted. The promotion
728 // prevents it from being picked up by the earlier bitcast case.
729 if (ValueVT.getVectorElementCount().isScalar() &&
730 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
731 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
732 DAG.getVectorIdxConstant(0, DL));
733 } else {
734 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
735 assert(PartVT.getFixedSizeInBits() > ValueSize &&
736 "lossy conversion of vector to scalar type");
737 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
738 Val = DAG.getBitcast(IntermediateType, Val);
739 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
740 }
741 }
742
743 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
744 Parts[0] = Val;
745 return;
746 }
747
748 // Handle a multi-element vector.
749 EVT IntermediateVT;
750 MVT RegisterVT;
751 unsigned NumIntermediates;
752 unsigned NumRegs;
753 if (IsABIRegCopy) {
755 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
756 RegisterVT);
757 } else {
758 NumRegs =
759 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
760 NumIntermediates, RegisterVT);
761 }
762
763 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
764 NumParts = NumRegs; // Silence a compiler warning.
765 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
766
767 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
768 "Mixing scalable and fixed vectors when copying in parts");
769
770 std::optional<ElementCount> DestEltCnt;
771
772 if (IntermediateVT.isVector())
773 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
774 else
775 DestEltCnt = ElementCount::getFixed(NumIntermediates);
776
777 EVT BuiltVectorTy = EVT::getVectorVT(
778 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
779
780 if (ValueVT == BuiltVectorTy) {
781 // Nothing to do.
782 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
783 // Bitconvert vector->vector case.
784 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
785 } else {
786 if (BuiltVectorTy.getVectorElementType().bitsGT(
787 ValueVT.getVectorElementType())) {
788 // Integer promotion.
789 ValueVT = EVT::getVectorVT(*DAG.getContext(),
790 BuiltVectorTy.getVectorElementType(),
791 ValueVT.getVectorElementCount());
792 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
793 }
794
795 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
796 Val = Widened;
797 }
798 }
799
800 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
801
802 // Split the vector into intermediate operands.
803 SmallVector<SDValue, 8> Ops(NumIntermediates);
804 for (unsigned i = 0; i != NumIntermediates; ++i) {
805 if (IntermediateVT.isVector()) {
806 // This does something sensible for scalable vectors - see the
807 // definition of EXTRACT_SUBVECTOR for further details.
808 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
809 Ops[i] =
810 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
811 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
812 } else {
813 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
814 DAG.getVectorIdxConstant(i, DL));
815 }
816 }
817
818 // Split the intermediate operands into legal parts.
819 if (NumParts == NumIntermediates) {
820 // If the register was not expanded, promote or copy the value,
821 // as appropriate.
822 for (unsigned i = 0; i != NumParts; ++i)
823 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
824 } else if (NumParts > 0) {
825 // If the intermediate type was expanded, split each the value into
826 // legal parts.
827 assert(NumIntermediates != 0 && "division by zero");
828 assert(NumParts % NumIntermediates == 0 &&
829 "Must expand into a divisible number of parts!");
830 unsigned Factor = NumParts / NumIntermediates;
831 for (unsigned i = 0; i != NumIntermediates; ++i)
832 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
833 CallConv);
834 }
835}
836
838 EVT valuevt, std::optional<CallingConv::ID> CC)
839 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
840 RegCount(1, regs.size()), CallConv(CC) {}
841
843 const DataLayout &DL, unsigned Reg, Type *Ty,
844 std::optional<CallingConv::ID> CC) {
845 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
846
847 CallConv = CC;
848
849 for (EVT ValueVT : ValueVTs) {
850 unsigned NumRegs =
853 : TLI.getNumRegisters(Context, ValueVT);
854 MVT RegisterVT =
857 : TLI.getRegisterType(Context, ValueVT);
858 for (unsigned i = 0; i != NumRegs; ++i)
859 Regs.push_back(Reg + i);
860 RegVTs.push_back(RegisterVT);
861 RegCount.push_back(NumRegs);
862 Reg += NumRegs;
863 }
864}
865
867 FunctionLoweringInfo &FuncInfo,
868 const SDLoc &dl, SDValue &Chain,
869 SDValue *Glue, const Value *V) const {
870 // A Value with type {} or [0 x %t] needs no registers.
871 if (ValueVTs.empty())
872 return SDValue();
873
874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
875
876 // Assemble the legal parts into the final values.
877 SmallVector<SDValue, 4> Values(ValueVTs.size());
879 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
880 // Copy the legal parts from the registers.
881 EVT ValueVT = ValueVTs[Value];
882 unsigned NumRegs = RegCount[Value];
883 MVT RegisterVT = isABIMangled()
885 *DAG.getContext(), *CallConv, RegVTs[Value])
886 : RegVTs[Value];
887
888 Parts.resize(NumRegs);
889 for (unsigned i = 0; i != NumRegs; ++i) {
890 SDValue P;
891 if (!Glue) {
892 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
893 } else {
894 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
895 *Glue = P.getValue(2);
896 }
897
898 Chain = P.getValue(1);
899 Parts[i] = P;
900
901 // If the source register was virtual and if we know something about it,
902 // add an assert node.
903 if (!Register::isVirtualRegister(Regs[Part + i]) ||
904 !RegisterVT.isInteger())
905 continue;
906
908 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
909 if (!LOI)
910 continue;
911
912 unsigned RegSize = RegisterVT.getScalarSizeInBits();
913 unsigned NumSignBits = LOI->NumSignBits;
914 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
915
916 if (NumZeroBits == RegSize) {
917 // The current value is a zero.
918 // Explicitly express that as it would be easier for
919 // optimizations to kick in.
920 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
921 continue;
922 }
923
924 // FIXME: We capture more information than the dag can represent. For
925 // now, just use the tightest assertzext/assertsext possible.
926 bool isSExt;
927 EVT FromVT(MVT::Other);
928 if (NumZeroBits) {
929 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
930 isSExt = false;
931 } else if (NumSignBits > 1) {
932 FromVT =
933 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
934 isSExt = true;
935 } else {
936 continue;
937 }
938 // Add an assertion node.
939 assert(FromVT != MVT::Other);
940 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
941 RegisterVT, P, DAG.getValueType(FromVT));
942 }
943
944 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
945 RegisterVT, ValueVT, V, Chain, CallConv);
946 Part += NumRegs;
947 Parts.clear();
948 }
949
950 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
951}
952
954 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
955 const Value *V,
956 ISD::NodeType PreferredExtendType) const {
957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
958 ISD::NodeType ExtendKind = PreferredExtendType;
959
960 // Get the list of the values's legal parts.
961 unsigned NumRegs = Regs.size();
962 SmallVector<SDValue, 8> Parts(NumRegs);
963 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
964 unsigned NumParts = RegCount[Value];
965
966 MVT RegisterVT = isABIMangled()
968 *DAG.getContext(), *CallConv, RegVTs[Value])
969 : RegVTs[Value];
970
971 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
972 ExtendKind = ISD::ZERO_EXTEND;
973
974 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
975 NumParts, RegisterVT, V, CallConv, ExtendKind);
976 Part += NumParts;
977 }
978
979 // Copy the parts into the registers.
980 SmallVector<SDValue, 8> Chains(NumRegs);
981 for (unsigned i = 0; i != NumRegs; ++i) {
982 SDValue Part;
983 if (!Glue) {
984 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
985 } else {
986 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
987 *Glue = Part.getValue(1);
988 }
989
990 Chains[i] = Part.getValue(0);
991 }
992
993 if (NumRegs == 1 || Glue)
994 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
995 // flagged to it. That is the CopyToReg nodes and the user are considered
996 // a single scheduling unit. If we create a TokenFactor and return it as
997 // chain, then the TokenFactor is both a predecessor (operand) of the
998 // user as well as a successor (the TF operands are flagged to the user).
999 // c1, f1 = CopyToReg
1000 // c2, f2 = CopyToReg
1001 // c3 = TokenFactor c1, c2
1002 // ...
1003 // = op c3, ..., f2
1004 Chain = Chains[NumRegs-1];
1005 else
1006 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1007}
1008
1010 unsigned MatchingIdx, const SDLoc &dl,
1011 SelectionDAG &DAG,
1012 std::vector<SDValue> &Ops) const {
1013 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1014
1015 InlineAsm::Flag Flag(Code, Regs.size());
1016 if (HasMatching)
1017 Flag.setMatchingOp(MatchingIdx);
1018 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1019 // Put the register class of the virtual registers in the flag word. That
1020 // way, later passes can recompute register class constraints for inline
1021 // assembly as well as normal instructions.
1022 // Don't do this for tied operands that can use the regclass information
1023 // from the def.
1025 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1026 Flag.setRegClass(RC->getID());
1027 }
1028
1029 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1030 Ops.push_back(Res);
1031
1032 if (Code == InlineAsm::Kind::Clobber) {
1033 // Clobbers should always have a 1:1 mapping with registers, and may
1034 // reference registers that have illegal (e.g. vector) types. Hence, we
1035 // shouldn't try to apply any sort of splitting logic to them.
1036 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1037 "No 1:1 mapping from clobbers to regs?");
1039 (void)SP;
1040 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1041 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1042 assert(
1043 (Regs[I] != SP ||
1045 "If we clobbered the stack pointer, MFI should know about it.");
1046 }
1047 return;
1048 }
1049
1050 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1051 MVT RegisterVT = RegVTs[Value];
1052 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1053 RegisterVT);
1054 for (unsigned i = 0; i != NumRegs; ++i) {
1055 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1056 unsigned TheReg = Regs[Reg++];
1057 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1058 }
1059 }
1060}
1061
1065 unsigned I = 0;
1066 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1067 unsigned RegCount = std::get<0>(CountAndVT);
1068 MVT RegisterVT = std::get<1>(CountAndVT);
1069 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1070 for (unsigned E = I + RegCount; I != E; ++I)
1071 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1072 }
1073 return OutVec;
1074}
1075
1077 AssumptionCache *ac,
1078 const TargetLibraryInfo *li) {
1079 AA = aa;
1080 AC = ac;
1081 GFI = gfi;
1082 LibInfo = li;
1084 LPadToCallSiteMap.clear();
1086 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1088}
1089
1091 NodeMap.clear();
1092 UnusedArgNodeMap.clear();
1093 PendingLoads.clear();
1094 PendingExports.clear();
1095 PendingConstrainedFP.clear();
1096 PendingConstrainedFPStrict.clear();
1097 CurInst = nullptr;
1098 HasTailCall = false;
1099 SDNodeOrder = LowestSDNodeOrder;
1101}
1102
1104 DanglingDebugInfoMap.clear();
1105}
1106
1107// Update DAG root to include dependencies on Pending chains.
1108SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1109 SDValue Root = DAG.getRoot();
1110
1111 if (Pending.empty())
1112 return Root;
1113
1114 // Add current root to PendingChains, unless we already indirectly
1115 // depend on it.
1116 if (Root.getOpcode() != ISD::EntryToken) {
1117 unsigned i = 0, e = Pending.size();
1118 for (; i != e; ++i) {
1119 assert(Pending[i].getNode()->getNumOperands() > 1);
1120 if (Pending[i].getNode()->getOperand(0) == Root)
1121 break; // Don't add the root if we already indirectly depend on it.
1122 }
1123
1124 if (i == e)
1125 Pending.push_back(Root);
1126 }
1127
1128 if (Pending.size() == 1)
1129 Root = Pending[0];
1130 else
1131 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1132
1133 DAG.setRoot(Root);
1134 Pending.clear();
1135 return Root;
1136}
1137
1139 return updateRoot(PendingLoads);
1140}
1141
1143 // Chain up all pending constrained intrinsics together with all
1144 // pending loads, by simply appending them to PendingLoads and
1145 // then calling getMemoryRoot().
1146 PendingLoads.reserve(PendingLoads.size() +
1147 PendingConstrainedFP.size() +
1148 PendingConstrainedFPStrict.size());
1149 PendingLoads.append(PendingConstrainedFP.begin(),
1150 PendingConstrainedFP.end());
1151 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1152 PendingConstrainedFPStrict.end());
1153 PendingConstrainedFP.clear();
1154 PendingConstrainedFPStrict.clear();
1155 return getMemoryRoot();
1156}
1157
1159 // We need to emit pending fpexcept.strict constrained intrinsics,
1160 // so append them to the PendingExports list.
1161 PendingExports.append(PendingConstrainedFPStrict.begin(),
1162 PendingConstrainedFPStrict.end());
1163 PendingConstrainedFPStrict.clear();
1164 return updateRoot(PendingExports);
1165}
1166
1168 DILocalVariable *Variable,
1170 DebugLoc DL) {
1171 assert(Variable && "Missing variable");
1172
1173 // Check if address has undef value.
1174 if (!Address || isa<UndefValue>(Address) ||
1175 (Address->use_empty() && !isa<Argument>(Address))) {
1176 LLVM_DEBUG(
1177 dbgs()
1178 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1179 return;
1180 }
1181
1182 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1183
1184 SDValue &N = NodeMap[Address];
1185 if (!N.getNode() && isa<Argument>(Address))
1186 // Check unused arguments map.
1187 N = UnusedArgNodeMap[Address];
1188 SDDbgValue *SDV;
1189 if (N.getNode()) {
1190 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1191 Address = BCI->getOperand(0);
1192 // Parameters are handled specially.
1193 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1194 if (IsParameter && FINode) {
1195 // Byval parameter. We have a frame index at this point.
1196 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1197 /*IsIndirect*/ true, DL, SDNodeOrder);
1198 } else if (isa<Argument>(Address)) {
1199 // Address is an argument, so try to emit its dbg value using
1200 // virtual register info from the FuncInfo.ValueMap.
1201 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1202 FuncArgumentDbgValueKind::Declare, N);
1203 return;
1204 } else {
1205 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1206 true, DL, SDNodeOrder);
1207 }
1208 DAG.AddDbgValue(SDV, IsParameter);
1209 } else {
1210 // If Address is an argument then try to emit its dbg value using
1211 // virtual register info from the FuncInfo.ValueMap.
1212 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1213 FuncArgumentDbgValueKind::Declare, N)) {
1214 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1215 << " (could not emit func-arg dbg_value)\n");
1216 }
1217 }
1218 return;
1219}
1220
1222 // Add SDDbgValue nodes for any var locs here. Do so before updating
1223 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1224 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1225 // Add SDDbgValue nodes for any var locs here. Do so before updating
1226 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1227 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1228 It != End; ++It) {
1229 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1230 dropDanglingDebugInfo(Var, It->Expr);
1231 if (It->Values.isKillLocation(It->Expr)) {
1232 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1233 continue;
1234 }
1235 SmallVector<Value *> Values(It->Values.location_ops());
1236 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1237 It->Values.hasArgList())) {
1239 for (Value *V : It->Values.location_ops())
1240 Vals.push_back(V);
1242 FnVarLocs->getDILocalVariable(It->VariableID),
1243 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1244 }
1245 }
1246 }
1247
1248 // We must skip DbgVariableRecords if they've already been processed above as
1249 // we have just emitted the debug values resulting from assignment tracking
1250 // analysis, making any existing DbgVariableRecords redundant (and probably
1251 // less correct). We still need to process DbgLabelRecords. This does sink
1252 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1253 // be important as it does so deterministcally and ordering between
1254 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1255 // printing).
1256 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1257 // Is there is any debug-info attached to this instruction, in the form of
1258 // DbgRecord non-instruction debug-info records.
1259 for (DbgRecord &DR : I.getDbgRecordRange()) {
1260 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1261 assert(DLR->getLabel() && "Missing label");
1262 SDDbgLabel *SDV =
1263 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1264 DAG.AddDbgLabel(SDV);
1265 continue;
1266 }
1267
1268 if (SkipDbgVariableRecords)
1269 continue;
1270 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
1271 DILocalVariable *Variable = DVR.getVariable();
1274
1276 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1277 continue;
1278 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1279 << "\n");
1281 DVR.getDebugLoc());
1282 continue;
1283 }
1284
1285 // A DbgVariableRecord with no locations is a kill location.
1287 if (Values.empty()) {
1289 SDNodeOrder);
1290 continue;
1291 }
1292
1293 // A DbgVariableRecord with an undef or absent location is also a kill
1294 // location.
1295 if (llvm::any_of(Values,
1296 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1298 SDNodeOrder);
1299 continue;
1300 }
1301
1302 bool IsVariadic = DVR.hasArgList();
1303 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1304 SDNodeOrder, IsVariadic)) {
1305 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1306 DVR.getDebugLoc(), SDNodeOrder);
1307 }
1308 }
1309}
1310
1312 visitDbgInfo(I);
1313
1314 // Set up outgoing PHI node register values before emitting the terminator.
1315 if (I.isTerminator()) {
1316 HandlePHINodesInSuccessorBlocks(I.getParent());
1317 }
1318
1319 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1320 if (!isa<DbgInfoIntrinsic>(I))
1321 ++SDNodeOrder;
1322
1323 CurInst = &I;
1324
1325 // Set inserted listener only if required.
1326 bool NodeInserted = false;
1327 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1328 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1329 if (PCSectionsMD) {
1330 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1331 DAG, [&](SDNode *) { NodeInserted = true; });
1332 }
1333
1334 visit(I.getOpcode(), I);
1335
1336 if (!I.isTerminator() && !HasTailCall &&
1337 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1339
1340 // Handle metadata.
1341 if (PCSectionsMD) {
1342 auto It = NodeMap.find(&I);
1343 if (It != NodeMap.end()) {
1344 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1345 } else if (NodeInserted) {
1346 // This should not happen; if it does, don't let it go unnoticed so we can
1347 // fix it. Relevant visit*() function is probably missing a setValue().
1348 errs() << "warning: loosing !pcsections metadata ["
1349 << I.getModule()->getName() << "]\n";
1350 LLVM_DEBUG(I.dump());
1351 assert(false);
1352 }
1353 }
1354
1355 CurInst = nullptr;
1356}
1357
1358void SelectionDAGBuilder::visitPHI(const PHINode &) {
1359 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1360}
1361
1362void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1363 // Note: this doesn't use InstVisitor, because it has to work with
1364 // ConstantExpr's in addition to instructions.
1365 switch (Opcode) {
1366 default: llvm_unreachable("Unknown instruction type encountered!");
1367 // Build the switch statement using the Instruction.def file.
1368#define HANDLE_INST(NUM, OPCODE, CLASS) \
1369 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1370#include "llvm/IR/Instruction.def"
1371 }
1372}
1373
1375 DILocalVariable *Variable,
1376 DebugLoc DL, unsigned Order,
1379 // For variadic dbg_values we will now insert an undef.
1380 // FIXME: We can potentially recover these!
1382 for (const Value *V : Values) {
1383 auto *Undef = UndefValue::get(V->getType());
1385 }
1386 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1387 /*IsIndirect=*/false, DL, Order,
1388 /*IsVariadic=*/true);
1389 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1390 return true;
1391}
1392
1394 DILocalVariable *Var,
1395 DIExpression *Expr,
1396 bool IsVariadic, DebugLoc DL,
1397 unsigned Order) {
1398 if (IsVariadic) {
1399 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1400 return;
1401 }
1402 // TODO: Dangling debug info will eventually either be resolved or produce
1403 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1404 // between the original dbg.value location and its resolved DBG_VALUE,
1405 // which we should ideally fill with an extra Undef DBG_VALUE.
1406 assert(Values.size() == 1);
1407 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1408}
1409
1411 const DIExpression *Expr) {
1412 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1413 DIVariable *DanglingVariable = DDI.getVariable();
1414 DIExpression *DanglingExpr = DDI.getExpression();
1415 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1416 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1417 << printDDI(nullptr, DDI) << "\n");
1418 return true;
1419 }
1420 return false;
1421 };
1422
1423 for (auto &DDIMI : DanglingDebugInfoMap) {
1424 DanglingDebugInfoVector &DDIV = DDIMI.second;
1425
1426 // If debug info is to be dropped, run it through final checks to see
1427 // whether it can be salvaged.
1428 for (auto &DDI : DDIV)
1429 if (isMatchingDbgValue(DDI))
1430 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1431
1432 erase_if(DDIV, isMatchingDbgValue);
1433 }
1434}
1435
1436// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1437// generate the debug data structures now that we've seen its definition.
1439 SDValue Val) {
1440 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1441 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1442 return;
1443
1444 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1445 for (auto &DDI : DDIV) {
1446 DebugLoc DL = DDI.getDebugLoc();
1447 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1448 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1449 DILocalVariable *Variable = DDI.getVariable();
1450 DIExpression *Expr = DDI.getExpression();
1452 "Expected inlined-at fields to agree");
1453 SDDbgValue *SDV;
1454 if (Val.getNode()) {
1455 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1456 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1457 // we couldn't resolve it directly when examining the DbgValue intrinsic
1458 // in the first place we should not be more successful here). Unless we
1459 // have some test case that prove this to be correct we should avoid
1460 // calling EmitFuncArgumentDbgValue here.
1461 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1462 FuncArgumentDbgValueKind::Value, Val)) {
1463 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1464 << printDDI(V, DDI) << "\n");
1465 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1466 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1467 // inserted after the definition of Val when emitting the instructions
1468 // after ISel. An alternative could be to teach
1469 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1470 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1471 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1472 << ValSDNodeOrder << "\n");
1473 SDV = getDbgValue(Val, Variable, Expr, DL,
1474 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1475 DAG.AddDbgValue(SDV, false);
1476 } else
1477 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1478 << printDDI(V, DDI)
1479 << " in EmitFuncArgumentDbgValue\n");
1480 } else {
1481 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1482 << "\n");
1483 auto Undef = UndefValue::get(V->getType());
1484 auto SDV =
1485 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1486 DAG.AddDbgValue(SDV, false);
1487 }
1488 }
1489 DDIV.clear();
1490}
1491
1493 DanglingDebugInfo &DDI) {
1494 // TODO: For the variadic implementation, instead of only checking the fail
1495 // state of `handleDebugValue`, we need know specifically which values were
1496 // invalid, so that we attempt to salvage only those values when processing
1497 // a DIArgList.
1498 const Value *OrigV = V;
1499 DILocalVariable *Var = DDI.getVariable();
1500 DIExpression *Expr = DDI.getExpression();
1501 DebugLoc DL = DDI.getDebugLoc();
1502 unsigned SDOrder = DDI.getSDNodeOrder();
1503
1504 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1505 // that DW_OP_stack_value is desired.
1506 bool StackValue = true;
1507
1508 // Can this Value can be encoded without any further work?
1509 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1510 return;
1511
1512 // Attempt to salvage back through as many instructions as possible. Bail if
1513 // a non-instruction is seen, such as a constant expression or global
1514 // variable. FIXME: Further work could recover those too.
1515 while (isa<Instruction>(V)) {
1516 const Instruction &VAsInst = *cast<const Instruction>(V);
1517 // Temporary "0", awaiting real implementation.
1519 SmallVector<Value *, 4> AdditionalValues;
1520 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1521 Expr->getNumLocationOperands(), Ops,
1522 AdditionalValues);
1523 // If we cannot salvage any further, and haven't yet found a suitable debug
1524 // expression, bail out.
1525 if (!V)
1526 break;
1527
1528 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1529 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1530 // here for variadic dbg_values, remove that condition.
1531 if (!AdditionalValues.empty())
1532 break;
1533
1534 // New value and expr now represent this debuginfo.
1535 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1536
1537 // Some kind of simplification occurred: check whether the operand of the
1538 // salvaged debug expression can be encoded in this DAG.
1539 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1540 LLVM_DEBUG(
1541 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1542 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1543 return;
1544 }
1545 }
1546
1547 // This was the final opportunity to salvage this debug information, and it
1548 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1549 // any earlier variable location.
1550 assert(OrigV && "V shouldn't be null");
1551 auto *Undef = UndefValue::get(OrigV->getType());
1552 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1553 DAG.AddDbgValue(SDV, false);
1554 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1555 << printDDI(OrigV, DDI) << "\n");
1556}
1557
1559 DIExpression *Expr,
1560 DebugLoc DbgLoc,
1561 unsigned Order) {
1565 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1566 /*IsVariadic*/ false);
1567}
1568
1570 DILocalVariable *Var,
1571 DIExpression *Expr, DebugLoc DbgLoc,
1572 unsigned Order, bool IsVariadic) {
1573 if (Values.empty())
1574 return true;
1575
1576 // Filter EntryValue locations out early.
1577 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1578 return true;
1579
1580 SmallVector<SDDbgOperand> LocationOps;
1581 SmallVector<SDNode *> Dependencies;
1582 for (const Value *V : Values) {
1583 // Constant value.
1584 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1585 isa<ConstantPointerNull>(V)) {
1586 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1587 continue;
1588 }
1589
1590 // Look through IntToPtr constants.
1591 if (auto *CE = dyn_cast<ConstantExpr>(V))
1592 if (CE->getOpcode() == Instruction::IntToPtr) {
1593 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1594 continue;
1595 }
1596
1597 // If the Value is a frame index, we can create a FrameIndex debug value
1598 // without relying on the DAG at all.
1599 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1600 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1601 if (SI != FuncInfo.StaticAllocaMap.end()) {
1602 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1603 continue;
1604 }
1605 }
1606
1607 // Do not use getValue() in here; we don't want to generate code at
1608 // this point if it hasn't been done yet.
1609 SDValue N = NodeMap[V];
1610 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1611 N = UnusedArgNodeMap[V];
1612 if (N.getNode()) {
1613 // Only emit func arg dbg value for non-variadic dbg.values for now.
1614 if (!IsVariadic &&
1615 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1616 FuncArgumentDbgValueKind::Value, N))
1617 return true;
1618 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1619 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1620 // describe stack slot locations.
1621 //
1622 // Consider "int x = 0; int *px = &x;". There are two kinds of
1623 // interesting debug values here after optimization:
1624 //
1625 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1626 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1627 //
1628 // Both describe the direct values of their associated variables.
1629 Dependencies.push_back(N.getNode());
1630 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1631 continue;
1632 }
1633 LocationOps.emplace_back(
1634 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1635 continue;
1636 }
1637
1639 // Special rules apply for the first dbg.values of parameter variables in a
1640 // function. Identify them by the fact they reference Argument Values, that
1641 // they're parameters, and they are parameters of the current function. We
1642 // need to let them dangle until they get an SDNode.
1643 bool IsParamOfFunc =
1644 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1645 if (IsParamOfFunc)
1646 return false;
1647
1648 // The value is not used in this block yet (or it would have an SDNode).
1649 // We still want the value to appear for the user if possible -- if it has
1650 // an associated VReg, we can refer to that instead.
1651 auto VMI = FuncInfo.ValueMap.find(V);
1652 if (VMI != FuncInfo.ValueMap.end()) {
1653 unsigned Reg = VMI->second;
1654 // If this is a PHI node, it may be split up into several MI PHI nodes
1655 // (in FunctionLoweringInfo::set).
1656 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1657 V->getType(), std::nullopt);
1658 if (RFV.occupiesMultipleRegs()) {
1659 // FIXME: We could potentially support variadic dbg_values here.
1660 if (IsVariadic)
1661 return false;
1662 unsigned Offset = 0;
1663 unsigned BitsToDescribe = 0;
1664 if (auto VarSize = Var->getSizeInBits())
1665 BitsToDescribe = *VarSize;
1666 if (auto Fragment = Expr->getFragmentInfo())
1667 BitsToDescribe = Fragment->SizeInBits;
1668 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1669 // Bail out if all bits are described already.
1670 if (Offset >= BitsToDescribe)
1671 break;
1672 // TODO: handle scalable vectors.
1673 unsigned RegisterSize = RegAndSize.second;
1674 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1675 ? BitsToDescribe - Offset
1676 : RegisterSize;
1677 auto FragmentExpr = DIExpression::createFragmentExpression(
1678 Expr, Offset, FragmentSize);
1679 if (!FragmentExpr)
1680 continue;
1682 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1683 DAG.AddDbgValue(SDV, false);
1684 Offset += RegisterSize;
1685 }
1686 return true;
1687 }
1688 // We can use simple vreg locations for variadic dbg_values as well.
1689 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1690 continue;
1691 }
1692 // We failed to create a SDDbgOperand for V.
1693 return false;
1694 }
1695
1696 // We have created a SDDbgOperand for each Value in Values.
1697 // Should use Order instead of SDNodeOrder?
1698 assert(!LocationOps.empty());
1699 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1700 /*IsIndirect=*/false, DbgLoc,
1701 SDNodeOrder, IsVariadic);
1702 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1703 return true;
1704}
1705
1707 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1708 for (auto &Pair : DanglingDebugInfoMap)
1709 for (auto &DDI : Pair.second)
1710 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1712}
1713
1714/// getCopyFromRegs - If there was virtual register allocated for the value V
1715/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1718 SDValue Result;
1719
1720 if (It != FuncInfo.ValueMap.end()) {
1721 Register InReg = It->second;
1722
1724 DAG.getDataLayout(), InReg, Ty,
1725 std::nullopt); // This is not an ABI copy.
1726 SDValue Chain = DAG.getEntryNode();
1727 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1728 V);
1729 resolveDanglingDebugInfo(V, Result);
1730 }
1731
1732 return Result;
1733}
1734
1735/// getValue - Return an SDValue for the given Value.
1737 // If we already have an SDValue for this value, use it. It's important
1738 // to do this first, so that we don't create a CopyFromReg if we already
1739 // have a regular SDValue.
1740 SDValue &N = NodeMap[V];
1741 if (N.getNode()) return N;
1742
1743 // If there's a virtual register allocated and initialized for this
1744 // value, use it.
1745 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1746 return copyFromReg;
1747
1748 // Otherwise create a new SDValue and remember it.
1749 SDValue Val = getValueImpl(V);
1750 NodeMap[V] = Val;
1752 return Val;
1753}
1754
1755/// getNonRegisterValue - Return an SDValue for the given Value, but
1756/// don't look in FuncInfo.ValueMap for a virtual register.
1758 // If we already have an SDValue for this value, use it.
1759 SDValue &N = NodeMap[V];
1760 if (N.getNode()) {
1761 if (isIntOrFPConstant(N)) {
1762 // Remove the debug location from the node as the node is about to be used
1763 // in a location which may differ from the original debug location. This
1764 // is relevant to Constant and ConstantFP nodes because they can appear
1765 // as constant expressions inside PHI nodes.
1766 N->setDebugLoc(DebugLoc());
1767 }
1768 return N;
1769 }
1770
1771 // Otherwise create a new SDValue and remember it.
1772 SDValue Val = getValueImpl(V);
1773 NodeMap[V] = Val;
1775 return Val;
1776}
1777
1778/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1779/// Create an SDValue for the given value.
1782
1783 if (const Constant *C = dyn_cast<Constant>(V)) {
1784 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1785
1786 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1787 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1788
1789 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1790 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1791
1792 if (isa<ConstantPointerNull>(C)) {
1793 unsigned AS = V->getType()->getPointerAddressSpace();
1794 return DAG.getConstant(0, getCurSDLoc(),
1795 TLI.getPointerTy(DAG.getDataLayout(), AS));
1796 }
1797
1798 if (match(C, m_VScale()))
1799 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1800
1801 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1802 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1803
1804 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1805 return DAG.getUNDEF(VT);
1806
1807 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1808 visit(CE->getOpcode(), *CE);
1809 SDValue N1 = NodeMap[V];
1810 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1811 return N1;
1812 }
1813
1814 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1816 for (const Use &U : C->operands()) {
1817 SDNode *Val = getValue(U).getNode();
1818 // If the operand is an empty aggregate, there are no values.
1819 if (!Val) continue;
1820 // Add each leaf value from the operand to the Constants list
1821 // to form a flattened list of all the values.
1822 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1823 Constants.push_back(SDValue(Val, i));
1824 }
1825
1827 }
1828
1829 if (const ConstantDataSequential *CDS =
1830 dyn_cast<ConstantDataSequential>(C)) {
1832 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1833 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1834 // Add each leaf value from the operand to the Constants list
1835 // to form a flattened list of all the values.
1836 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1837 Ops.push_back(SDValue(Val, i));
1838 }
1839
1840 if (isa<ArrayType>(CDS->getType()))
1841 return DAG.getMergeValues(Ops, getCurSDLoc());
1842 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1843 }
1844
1845 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1846 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1847 "Unknown struct or array constant!");
1848
1849 SmallVector<EVT, 4> ValueVTs;
1850 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1851 unsigned NumElts = ValueVTs.size();
1852 if (NumElts == 0)
1853 return SDValue(); // empty struct
1855 for (unsigned i = 0; i != NumElts; ++i) {
1856 EVT EltVT = ValueVTs[i];
1857 if (isa<UndefValue>(C))
1858 Constants[i] = DAG.getUNDEF(EltVT);
1859 else if (EltVT.isFloatingPoint())
1860 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1861 else
1862 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1863 }
1864
1866 }
1867
1868 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1869 return DAG.getBlockAddress(BA, VT);
1870
1871 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1872 return getValue(Equiv->getGlobalValue());
1873
1874 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1875 return getValue(NC->getGlobalValue());
1876
1877 if (VT == MVT::aarch64svcount) {
1878 assert(C->isNullValue() && "Can only zero this target type!");
1879 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1880 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1881 }
1882
1883 VectorType *VecTy = cast<VectorType>(V->getType());
1884
1885 // Now that we know the number and type of the elements, get that number of
1886 // elements into the Ops array based on what kind of constant it is.
1887 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1889 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1890 for (unsigned i = 0; i != NumElements; ++i)
1891 Ops.push_back(getValue(CV->getOperand(i)));
1892
1893 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1894 }
1895
1896 if (isa<ConstantAggregateZero>(C)) {
1897 EVT EltVT =
1899
1900 SDValue Op;
1901 if (EltVT.isFloatingPoint())
1902 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1903 else
1904 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1905
1906 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1907 }
1908
1909 llvm_unreachable("Unknown vector constant");
1910 }
1911
1912 // If this is a static alloca, generate it as the frameindex instead of
1913 // computation.
1914 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1916 FuncInfo.StaticAllocaMap.find(AI);
1917 if (SI != FuncInfo.StaticAllocaMap.end())
1918 return DAG.getFrameIndex(
1919 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1920 }
1921
1922 // If this is an instruction which fast-isel has deferred, select it now.
1923 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1925
1926 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1927 Inst->getType(), std::nullopt);
1928 SDValue Chain = DAG.getEntryNode();
1929 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1930 }
1931
1932 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1933 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1934
1935 if (const auto *BB = dyn_cast<BasicBlock>(V))
1936 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1937
1938 llvm_unreachable("Can't get register for value!");
1939}
1940
1941void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1943 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1944 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1945 bool IsSEH = isAsynchronousEHPersonality(Pers);
1946 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1947 if (!IsSEH)
1948 CatchPadMBB->setIsEHScopeEntry();
1949 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1950 if (IsMSVCCXX || IsCoreCLR)
1951 CatchPadMBB->setIsEHFuncletEntry();
1952}
1953
1954void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1955 // Update machine-CFG edge.
1956 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1957 FuncInfo.MBB->addSuccessor(TargetMBB);
1958 TargetMBB->setIsEHCatchretTarget(true);
1960
1962 bool IsSEH = isAsynchronousEHPersonality(Pers);
1963 if (IsSEH) {
1964 // If this is not a fall-through branch or optimizations are switched off,
1965 // emit the branch.
1966 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1968 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1969 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1970 return;
1971 }
1972
1973 // Figure out the funclet membership for the catchret's successor.
1974 // This will be used by the FuncletLayout pass to determine how to order the
1975 // BB's.
1976 // A 'catchret' returns to the outer scope's color.
1977 Value *ParentPad = I.getCatchSwitchParentPad();
1978 const BasicBlock *SuccessorColor;
1979 if (isa<ConstantTokenNone>(ParentPad))
1980 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1981 else
1982 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1983 assert(SuccessorColor && "No parent funclet for catchret!");
1984 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1985 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1986
1987 // Create the terminator node.
1989 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1990 DAG.getBasicBlock(SuccessorColorMBB));
1991 DAG.setRoot(Ret);
1992}
1993
1994void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1995 // Don't emit any special code for the cleanuppad instruction. It just marks
1996 // the start of an EH scope/funclet.
1999 if (Pers != EHPersonality::Wasm_CXX) {
2002 }
2003}
2004
2005// In wasm EH, even though a catchpad may not catch an exception if a tag does
2006// not match, it is OK to add only the first unwind destination catchpad to the
2007// successors, because there will be at least one invoke instruction within the
2008// catch scope that points to the next unwind destination, if one exists, so
2009// CFGSort cannot mess up with BB sorting order.
2010// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2011// call within them, and catchpads only consisting of 'catch (...)' have a
2012// '__cxa_end_catch' call within them, both of which generate invokes in case
2013// the next unwind destination exists, i.e., the next unwind destination is not
2014// the caller.)
2015//
2016// Having at most one EH pad successor is also simpler and helps later
2017// transformations.
2018//
2019// For example,
2020// current:
2021// invoke void @foo to ... unwind label %catch.dispatch
2022// catch.dispatch:
2023// %0 = catchswitch within ... [label %catch.start] unwind label %next
2024// catch.start:
2025// ...
2026// ... in this BB or some other child BB dominated by this BB there will be an
2027// invoke that points to 'next' BB as an unwind destination
2028//
2029// next: ; We don't need to add this to 'current' BB's successor
2030// ...
2032 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2033 BranchProbability Prob,
2034 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2035 &UnwindDests) {
2036 while (EHPadBB) {
2037 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2038 if (isa<CleanupPadInst>(Pad)) {
2039 // Stop on cleanup pads.
2040 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2041 UnwindDests.back().first->setIsEHScopeEntry();
2042 break;
2043 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2044 // Add the catchpad handlers to the possible destinations. We don't
2045 // continue to the unwind destination of the catchswitch for wasm.
2046 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2047 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2048 UnwindDests.back().first->setIsEHScopeEntry();
2049 }
2050 break;
2051 } else {
2052 continue;
2053 }
2054 }
2055}
2056
2057/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2058/// many places it could ultimately go. In the IR, we have a single unwind
2059/// destination, but in the machine CFG, we enumerate all the possible blocks.
2060/// This function skips over imaginary basic blocks that hold catchswitch
2061/// instructions, and finds all the "real" machine
2062/// basic block destinations. As those destinations may not be successors of
2063/// EHPadBB, here we also calculate the edge probability to those destinations.
2064/// The passed-in Prob is the edge probability to EHPadBB.
2066 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2067 BranchProbability Prob,
2068 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2069 &UnwindDests) {
2070 EHPersonality Personality =
2072 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2073 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2074 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2075 bool IsSEH = isAsynchronousEHPersonality(Personality);
2076
2077 if (IsWasmCXX) {
2078 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2079 assert(UnwindDests.size() <= 1 &&
2080 "There should be at most one unwind destination for wasm");
2081 return;
2082 }
2083
2084 while (EHPadBB) {
2085 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2086 BasicBlock *NewEHPadBB = nullptr;
2087 if (isa<LandingPadInst>(Pad)) {
2088 // Stop on landingpads. They are not funclets.
2089 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2090 break;
2091 } else if (isa<CleanupPadInst>(Pad)) {
2092 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2093 // personalities.
2094 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2095 UnwindDests.back().first->setIsEHScopeEntry();
2096 UnwindDests.back().first->setIsEHFuncletEntry();
2097 break;
2098 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2099 // Add the catchpad handlers to the possible destinations.
2100 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2101 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2102 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2103 if (IsMSVCCXX || IsCoreCLR)
2104 UnwindDests.back().first->setIsEHFuncletEntry();
2105 if (!IsSEH)
2106 UnwindDests.back().first->setIsEHScopeEntry();
2107 }
2108 NewEHPadBB = CatchSwitch->getUnwindDest();
2109 } else {
2110 continue;
2111 }
2112
2113 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2114 if (BPI && NewEHPadBB)
2115 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2116 EHPadBB = NewEHPadBB;
2117 }
2118}
2119
2120void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2121 // Update successor info.
2123 auto UnwindDest = I.getUnwindDest();
2125 BranchProbability UnwindDestProb =
2126 (BPI && UnwindDest)
2127 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2129 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2130 for (auto &UnwindDest : UnwindDests) {
2131 UnwindDest.first->setIsEHPad();
2132 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2133 }
2135
2136 // Create the terminator node.
2137 SDValue Ret =
2139 DAG.setRoot(Ret);
2140}
2141
2142void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2143 report_fatal_error("visitCatchSwitch not yet implemented!");
2144}
2145
2146void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2148 auto &DL = DAG.getDataLayout();
2149 SDValue Chain = getControlRoot();
2152
2153 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2154 // lower
2155 //
2156 // %val = call <ty> @llvm.experimental.deoptimize()
2157 // ret <ty> %val
2158 //
2159 // differently.
2160 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2162 return;
2163 }
2164
2165 if (!FuncInfo.CanLowerReturn) {
2166 unsigned DemoteReg = FuncInfo.DemoteRegister;
2167 const Function *F = I.getParent()->getParent();
2168
2169 // Emit a store of the return value through the virtual register.
2170 // Leave Outs empty so that LowerReturn won't try to load return
2171 // registers the usual way.
2172 SmallVector<EVT, 1> PtrValueVTs;
2173 ComputeValueVTs(TLI, DL,
2174 PointerType::get(F->getContext(),
2176 PtrValueVTs);
2177
2178 SDValue RetPtr =
2179 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2180 SDValue RetOp = getValue(I.getOperand(0));
2181
2182 SmallVector<EVT, 4> ValueVTs, MemVTs;
2184 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2185 &Offsets, 0);
2186 unsigned NumValues = ValueVTs.size();
2187
2188 SmallVector<SDValue, 4> Chains(NumValues);
2189 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2190 for (unsigned i = 0; i != NumValues; ++i) {
2191 // An aggregate return value cannot wrap around the address space, so
2192 // offsets to its parts don't wrap either.
2194 TypeSize::getFixed(Offsets[i]));
2195
2196 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2197 if (MemVTs[i] != ValueVTs[i])
2198 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2199 Chains[i] = DAG.getStore(
2200 Chain, getCurSDLoc(), Val,
2201 // FIXME: better loc info would be nice.
2203 commonAlignment(BaseAlign, Offsets[i]));
2204 }
2205
2207 MVT::Other, Chains);
2208 } else if (I.getNumOperands() != 0) {
2209 SmallVector<EVT, 4> ValueVTs;
2210 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2211 unsigned NumValues = ValueVTs.size();
2212 if (NumValues) {
2213 SDValue RetOp = getValue(I.getOperand(0));
2214
2215 const Function *F = I.getParent()->getParent();
2216
2217 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2218 I.getOperand(0)->getType(), F->getCallingConv(),
2219 /*IsVarArg*/ false, DL);
2220
2221 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2222 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2223 ExtendKind = ISD::SIGN_EXTEND;
2224 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2225 ExtendKind = ISD::ZERO_EXTEND;
2226
2227 LLVMContext &Context = F->getContext();
2228 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2229
2230 for (unsigned j = 0; j != NumValues; ++j) {
2231 EVT VT = ValueVTs[j];
2232
2233 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2234 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2235
2236 CallingConv::ID CC = F->getCallingConv();
2237
2238 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2239 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2240 SmallVector<SDValue, 4> Parts(NumParts);
2242 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2243 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2244
2245 // 'inreg' on function refers to return value
2247 if (RetInReg)
2248 Flags.setInReg();
2249
2250 if (I.getOperand(0)->getType()->isPointerTy()) {
2251 Flags.setPointer();
2252 Flags.setPointerAddrSpace(
2253 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2254 }
2255
2256 if (NeedsRegBlock) {
2257 Flags.setInConsecutiveRegs();
2258 if (j == NumValues - 1)
2259 Flags.setInConsecutiveRegsLast();
2260 }
2261
2262 // Propagate extension type if any
2263 if (ExtendKind == ISD::SIGN_EXTEND)
2264 Flags.setSExt();
2265 else if (ExtendKind == ISD::ZERO_EXTEND)
2266 Flags.setZExt();
2267
2268 for (unsigned i = 0; i < NumParts; ++i) {
2269 Outs.push_back(ISD::OutputArg(Flags,
2270 Parts[i].getValueType().getSimpleVT(),
2271 VT, /*isfixed=*/true, 0, 0));
2272 OutVals.push_back(Parts[i]);
2273 }
2274 }
2275 }
2276 }
2277
2278 // Push in swifterror virtual register as the last element of Outs. This makes
2279 // sure swifterror virtual register will be returned in the swifterror
2280 // physical register.
2281 const Function *F = I.getParent()->getParent();
2282 if (TLI.supportSwiftError() &&
2283 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2284 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2286 Flags.setSwiftError();
2288 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2289 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2290 // Create SDNode for the swifterror virtual register.
2291 OutVals.push_back(
2294 EVT(TLI.getPointerTy(DL))));
2295 }
2296
2297 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2298 CallingConv::ID CallConv =
2301 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2302
2303 // Verify that the target's LowerReturn behaved as expected.
2304 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2305 "LowerReturn didn't return a valid chain!");
2306
2307 // Update the DAG with the new chain value resulting from return lowering.
2308 DAG.setRoot(Chain);
2309}
2310
2311/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2312/// created for it, emit nodes to copy the value into the virtual
2313/// registers.
2315 // Skip empty types
2316 if (V->getType()->isEmptyTy())
2317 return;
2318
2320 if (VMI != FuncInfo.ValueMap.end()) {
2321 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2322 "Unused value assigned virtual registers!");
2323 CopyValueToVirtualRegister(V, VMI->second);
2324 }
2325}
2326
2327/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2328/// the current basic block, add it to ValueMap now so that we'll get a
2329/// CopyTo/FromReg.
2331 // No need to export constants.
2332 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2333
2334 // Already exported?
2335 if (FuncInfo.isExportedInst(V)) return;
2336
2339}
2340
2342 const BasicBlock *FromBB) {
2343 // The operands of the setcc have to be in this block. We don't know
2344 // how to export them from some other block.
2345 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2346 // Can export from current BB.
2347 if (VI->getParent() == FromBB)
2348 return true;
2349
2350 // Is already exported, noop.
2351 return FuncInfo.isExportedInst(V);
2352 }
2353
2354 // If this is an argument, we can export it if the BB is the entry block or
2355 // if it is already exported.
2356 if (isa<Argument>(V)) {
2357 if (FromBB->isEntryBlock())
2358 return true;
2359
2360 // Otherwise, can only export this if it is already exported.
2361 return FuncInfo.isExportedInst(V);
2362 }
2363
2364 // Otherwise, constants can always be exported.
2365 return true;
2366}
2367
2368/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2370SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2371 const MachineBasicBlock *Dst) const {
2373 const BasicBlock *SrcBB = Src->getBasicBlock();
2374 const BasicBlock *DstBB = Dst->getBasicBlock();
2375 if (!BPI) {
2376 // If BPI is not available, set the default probability as 1 / N, where N is
2377 // the number of successors.
2378 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2379 return BranchProbability(1, SuccSize);
2380 }
2381 return BPI->getEdgeProbability(SrcBB, DstBB);
2382}
2383
2384void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2385 MachineBasicBlock *Dst,
2386 BranchProbability Prob) {
2387 if (!FuncInfo.BPI)
2388 Src->addSuccessorWithoutProb(Dst);
2389 else {
2390 if (Prob.isUnknown())
2391 Prob = getEdgeProbability(Src, Dst);
2392 Src->addSuccessor(Dst, Prob);
2393 }
2394}
2395
2396static bool InBlock(const Value *V, const BasicBlock *BB) {
2397 if (const Instruction *I = dyn_cast<Instruction>(V))
2398 return I->getParent() == BB;
2399 return true;
2400}
2401
2402/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2403/// This function emits a branch and is used at the leaves of an OR or an
2404/// AND operator tree.
2405void
2408 MachineBasicBlock *FBB,
2409 MachineBasicBlock *CurBB,
2410 MachineBasicBlock *SwitchBB,
2411 BranchProbability TProb,
2412 BranchProbability FProb,
2413 bool InvertCond) {
2414 const BasicBlock *BB = CurBB->getBasicBlock();
2415
2416 // If the leaf of the tree is a comparison, merge the condition into
2417 // the caseblock.
2418 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2419 // The operands of the cmp have to be in this block. We don't know
2420 // how to export them from some other block. If this is the first block
2421 // of the sequence, no exporting is needed.
2422 if (CurBB == SwitchBB ||
2423 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2424 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2425 ISD::CondCode Condition;
2426 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2427 ICmpInst::Predicate Pred =
2428 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2429 Condition = getICmpCondCode(Pred);
2430 } else {
2431 const FCmpInst *FC = cast<FCmpInst>(Cond);
2432 FCmpInst::Predicate Pred =
2433 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2434 Condition = getFCmpCondCode(Pred);
2435 if (TM.Options.NoNaNsFPMath)
2436 Condition = getFCmpCodeWithoutNaN(Condition);
2437 }
2438
2439 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2440 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2441 SL->SwitchCases.push_back(CB);
2442 return;
2443 }
2444 }
2445
2446 // Create a CaseBlock record representing this branch.
2447 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2449 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2450 SL->SwitchCases.push_back(CB);
2451}
2452
2453// Collect dependencies on V recursively. This is used for the cost analysis in
2454// `shouldKeepJumpConditionsTogether`.
2458 unsigned Depth = 0) {
2459 // Return false if we have an incomplete count.
2461 return false;
2462
2463 auto *I = dyn_cast<Instruction>(V);
2464 if (I == nullptr)
2465 return true;
2466
2467 if (Necessary != nullptr) {
2468 // This instruction is necessary for the other side of the condition so
2469 // don't count it.
2470 if (Necessary->contains(I))
2471 return true;
2472 }
2473
2474 // Already added this dep.
2475 if (!Deps->try_emplace(I, false).second)
2476 return true;
2477
2478 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2479 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2480 Depth + 1))
2481 return false;
2482 return true;
2483}
2484
2486 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2487 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2489 if (I.getNumSuccessors() != 2)
2490 return false;
2491
2492 if (!I.isConditional())
2493 return false;
2494
2495 if (Params.BaseCost < 0)
2496 return false;
2497
2498 // Baseline cost.
2499 InstructionCost CostThresh = Params.BaseCost;
2500
2501 BranchProbabilityInfo *BPI = nullptr;
2502 if (Params.LikelyBias || Params.UnlikelyBias)
2503 BPI = FuncInfo.BPI;
2504 if (BPI != nullptr) {
2505 // See if we are either likely to get an early out or compute both lhs/rhs
2506 // of the condition.
2507 BasicBlock *IfFalse = I.getSuccessor(0);
2508 BasicBlock *IfTrue = I.getSuccessor(1);
2509
2510 std::optional<bool> Likely;
2511 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2512 Likely = true;
2513 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2514 Likely = false;
2515
2516 if (Likely) {
2517 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2518 // Its likely we will have to compute both lhs and rhs of condition
2519 CostThresh += Params.LikelyBias;
2520 else {
2521 if (Params.UnlikelyBias < 0)
2522 return false;
2523 // Its likely we will get an early out.
2524 CostThresh -= Params.UnlikelyBias;
2525 }
2526 }
2527 }
2528
2529 if (CostThresh <= 0)
2530 return false;
2531
2532 // Collect "all" instructions that lhs condition is dependent on.
2533 // Use map for stable iteration (to avoid non-determanism of iteration of
2534 // SmallPtrSet). The `bool` value is just a dummy.
2536 collectInstructionDeps(&LhsDeps, Lhs);
2537 // Collect "all" instructions that rhs condition is dependent on AND are
2538 // dependencies of lhs. This gives us an estimate on which instructions we
2539 // stand to save by splitting the condition.
2540 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2541 return false;
2542 // Add the compare instruction itself unless its a dependency on the LHS.
2543 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2544 if (!LhsDeps.contains(RhsI))
2545 RhsDeps.try_emplace(RhsI, false);
2546
2547 const auto &TLI = DAG.getTargetLoweringInfo();
2548 const auto &TTI =
2549 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2550
2551 InstructionCost CostOfIncluding = 0;
2552 // See if this instruction will need to computed independently of whether RHS
2553 // is.
2554 Value *BrCond = I.getCondition();
2555 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2556 for (const auto *U : Ins->users()) {
2557 // If user is independent of RHS calculation we don't need to count it.
2558 if (auto *UIns = dyn_cast<Instruction>(U))
2559 if (UIns != BrCond && !RhsDeps.contains(UIns))
2560 return false;
2561 }
2562 return true;
2563 };
2564
2565 // Prune instructions from RHS Deps that are dependencies of unrelated
2566 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2567 // arbitrary and just meant to cap the how much time we spend in the pruning
2568 // loop. Its highly unlikely to come into affect.
2569 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2570 // Stop after a certain point. No incorrectness from including too many
2571 // instructions.
2572 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2573 const Instruction *ToDrop = nullptr;
2574 for (const auto &InsPair : RhsDeps) {
2575 if (!ShouldCountInsn(InsPair.first)) {
2576 ToDrop = InsPair.first;
2577 break;
2578 }
2579 }
2580 if (ToDrop == nullptr)
2581 break;
2582 RhsDeps.erase(ToDrop);
2583 }
2584
2585 for (const auto &InsPair : RhsDeps) {
2586 // Finally accumulate latency that we can only attribute to computing the
2587 // RHS condition. Use latency because we are essentially trying to calculate
2588 // the cost of the dependency chain.
2589 // Possible TODO: We could try to estimate ILP and make this more precise.
2590 CostOfIncluding +=
2592
2593 if (CostOfIncluding > CostThresh)
2594 return false;
2595 }
2596 return true;
2597}
2598
2601 MachineBasicBlock *FBB,
2602 MachineBasicBlock *CurBB,
2603 MachineBasicBlock *SwitchBB,
2605 BranchProbability TProb,
2606 BranchProbability FProb,
2607 bool InvertCond) {
2608 // Skip over not part of the tree and remember to invert op and operands at
2609 // next level.
2610 Value *NotCond;
2611 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2612 InBlock(NotCond, CurBB->getBasicBlock())) {
2613 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2614 !InvertCond);
2615 return;
2616 }
2617
2618 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2619 const Value *BOpOp0, *BOpOp1;
2620 // Compute the effective opcode for Cond, taking into account whether it needs
2621 // to be inverted, e.g.
2622 // and (not (or A, B)), C
2623 // gets lowered as
2624 // and (and (not A, not B), C)
2626 if (BOp) {
2627 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2628 ? Instruction::And
2629 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2630 ? Instruction::Or
2632 if (InvertCond) {
2633 if (BOpc == Instruction::And)
2634 BOpc = Instruction::Or;
2635 else if (BOpc == Instruction::Or)
2636 BOpc = Instruction::And;
2637 }
2638 }
2639
2640 // If this node is not part of the or/and tree, emit it as a branch.
2641 // Note that all nodes in the tree should have same opcode.
2642 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2643 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2644 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2645 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2646 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2647 TProb, FProb, InvertCond);
2648 return;
2649 }
2650
2651 // Create TmpBB after CurBB.
2652 MachineFunction::iterator BBI(CurBB);
2655 CurBB->getParent()->insert(++BBI, TmpBB);
2656
2657 if (Opc == Instruction::Or) {
2658 // Codegen X | Y as:
2659 // BB1:
2660 // jmp_if_X TBB
2661 // jmp TmpBB
2662 // TmpBB:
2663 // jmp_if_Y TBB
2664 // jmp FBB
2665 //
2666
2667 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2668 // The requirement is that
2669 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2670 // = TrueProb for original BB.
2671 // Assuming the original probabilities are A and B, one choice is to set
2672 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2673 // A/(1+B) and 2B/(1+B). This choice assumes that
2674 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2675 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2676 // TmpBB, but the math is more complicated.
2677
2678 auto NewTrueProb = TProb / 2;
2679 auto NewFalseProb = TProb / 2 + FProb;
2680 // Emit the LHS condition.
2681 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2682 NewFalseProb, InvertCond);
2683
2684 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2685 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2686 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2687 // Emit the RHS condition into TmpBB.
2688 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2689 Probs[1], InvertCond);
2690 } else {
2691 assert(Opc == Instruction::And && "Unknown merge op!");
2692 // Codegen X & Y as:
2693 // BB1:
2694 // jmp_if_X TmpBB
2695 // jmp FBB
2696 // TmpBB:
2697 // jmp_if_Y TBB
2698 // jmp FBB
2699 //
2700 // This requires creation of TmpBB after CurBB.
2701
2702 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2703 // The requirement is that
2704 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2705 // = FalseProb for original BB.
2706 // Assuming the original probabilities are A and B, one choice is to set
2707 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2708 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2709 // TrueProb for BB1 * FalseProb for TmpBB.
2710
2711 auto NewTrueProb = TProb + FProb / 2;
2712 auto NewFalseProb = FProb / 2;
2713 // Emit the LHS condition.
2714 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2715 NewFalseProb, InvertCond);
2716
2717 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2718 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2719 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2720 // Emit the RHS condition into TmpBB.
2721 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2722 Probs[1], InvertCond);
2723 }
2724}
2725
2726/// If the set of cases should be emitted as a series of branches, return true.
2727/// If we should emit this as a bunch of and/or'd together conditions, return
2728/// false.
2729bool
2730SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2731 if (Cases.size() != 2) return true;
2732
2733 // If this is two comparisons of the same values or'd or and'd together, they
2734 // will get folded into a single comparison, so don't emit two blocks.
2735 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2736 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2737 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2738 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2739 return false;
2740 }
2741
2742 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2743 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2744 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2745 Cases[0].CC == Cases[1].CC &&
2746 isa<Constant>(Cases[0].CmpRHS) &&
2747 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2748 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2749 return false;
2750 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2751 return false;
2752 }
2753
2754 return true;
2755}
2756
2757void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2759
2760 // Update machine-CFG edges.
2761 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2762
2763 if (I.isUnconditional()) {
2764 // Update machine-CFG edges.
2765 BrMBB->addSuccessor(Succ0MBB);
2766
2767 // If this is not a fall-through branch or optimizations are switched off,
2768 // emit the branch.
2769 if (Succ0MBB != NextBlock(BrMBB) ||
2771 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2772 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2773 setValue(&I, Br);
2774 DAG.setRoot(Br);
2775 }
2776
2777 return;
2778 }
2779
2780 // If this condition is one of the special cases we handle, do special stuff
2781 // now.
2782 const Value *CondVal = I.getCondition();
2783 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2784
2785 // If this is a series of conditions that are or'd or and'd together, emit
2786 // this as a sequence of branches instead of setcc's with and/or operations.
2787 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2788 // unpredictable branches, and vector extracts because those jumps are likely
2789 // expensive for any target), this should improve performance.
2790 // For example, instead of something like:
2791 // cmp A, B
2792 // C = seteq
2793 // cmp D, E
2794 // F = setle
2795 // or C, F
2796 // jnz foo
2797 // Emit:
2798 // cmp A, B
2799 // je foo
2800 // cmp D, E
2801 // jle foo
2802 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2803 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2804 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2805 Value *Vec;
2806 const Value *BOp0, *BOp1;
2808 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2809 Opcode = Instruction::And;
2810 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2811 Opcode = Instruction::Or;
2812
2813 if (Opcode &&
2814 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2815 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2817 FuncInfo, I, Opcode, BOp0, BOp1,
2819 Opcode, BOp0, BOp1))) {
2820 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2821 getEdgeProbability(BrMBB, Succ0MBB),
2822 getEdgeProbability(BrMBB, Succ1MBB),
2823 /*InvertCond=*/false);
2824 // If the compares in later blocks need to use values not currently
2825 // exported from this block, export them now. This block should always
2826 // be the first entry.
2827 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2828
2829 // Allow some cases to be rejected.
2830 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2831 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2832 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2833 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2834 }
2835
2836 // Emit the branch for this block.
2837 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2838 SL->SwitchCases.erase(SL->SwitchCases.begin());
2839 return;
2840 }
2841
2842 // Okay, we decided not to do this, remove any inserted MBB's and clear
2843 // SwitchCases.
2844 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2845 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2846
2847 SL->SwitchCases.clear();
2848 }
2849 }
2850
2851 // Create a CaseBlock record representing this branch.
2853 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2854
2855 // Use visitSwitchCase to actually insert the fast branch sequence for this
2856 // cond branch.
2857 visitSwitchCase(CB, BrMBB);
2858}
2859
2860/// visitSwitchCase - Emits the necessary code to represent a single node in
2861/// the binary search tree resulting from lowering a switch instruction.
2863 MachineBasicBlock *SwitchBB) {
2864 SDValue Cond;
2865 SDValue CondLHS = getValue(CB.CmpLHS);
2866 SDLoc dl = CB.DL;
2867
2868 if (CB.CC == ISD::SETTRUE) {
2869 // Branch or fall through to TrueBB.
2870 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2871 SwitchBB->normalizeSuccProbs();
2872 if (CB.TrueBB != NextBlock(SwitchBB)) {
2873 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2874 DAG.getBasicBlock(CB.TrueBB)));
2875 }
2876 return;
2877 }
2878
2879 auto &TLI = DAG.getTargetLoweringInfo();
2880 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2881
2882 // Build the setcc now.
2883 if (!CB.CmpMHS) {
2884 // Fold "(X == true)" to X and "(X == false)" to !X to
2885 // handle common cases produced by branch lowering.
2886 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2887 CB.CC == ISD::SETEQ)
2888 Cond = CondLHS;
2889 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2890 CB.CC == ISD::SETEQ) {
2891 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2892 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2893 } else {
2894 SDValue CondRHS = getValue(CB.CmpRHS);
2895
2896 // If a pointer's DAG type is larger than its memory type then the DAG
2897 // values are zero-extended. This breaks signed comparisons so truncate
2898 // back to the underlying type before doing the compare.
2899 if (CondLHS.getValueType() != MemVT) {
2900 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2901 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2902 }
2903 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2904 }
2905 } else {
2906 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2907
2908 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2909 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2910
2911 SDValue CmpOp = getValue(CB.CmpMHS);
2912 EVT VT = CmpOp.getValueType();
2913
2914 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2915 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2916 ISD::SETLE);
2917 } else {
2918 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2919 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2920 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2921 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2922 }
2923 }
2924
2925 // Update successor info
2926 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2927 // TrueBB and FalseBB are always different unless the incoming IR is
2928 // degenerate. This only happens when running llc on weird IR.
2929 if (CB.TrueBB != CB.FalseBB)
2930 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2931 SwitchBB->normalizeSuccProbs();
2932
2933 // If the lhs block is the next block, invert the condition so that we can
2934 // fall through to the lhs instead of the rhs block.
2935 if (CB.TrueBB == NextBlock(SwitchBB)) {
2936 std::swap(CB.TrueBB, CB.FalseBB);
2937 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2938 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2939 }
2940
2941 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2942 MVT::Other, getControlRoot(), Cond,
2944
2945 setValue(CurInst, BrCond);
2946
2947 // Insert the false branch. Do this even if it's a fall through branch,
2948 // this makes it easier to do DAG optimizations which require inverting
2949 // the branch condition.
2950 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2952
2953 DAG.setRoot(BrCond);
2954}
2955
2956/// visitJumpTable - Emit JumpTable node in the current MBB
2958 // Emit the code for the jump table
2959 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2960 assert(JT.Reg != -1U && "Should lower JT Header first!");
2962 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2963 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2964 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2965 Index.getValue(1), Table, Index);
2966 DAG.setRoot(BrJumpTable);
2967}
2968
2969/// visitJumpTableHeader - This function emits necessary code to produce index
2970/// in the JumpTable from switch case.
2972 JumpTableHeader &JTH,
2973 MachineBasicBlock *SwitchBB) {
2974 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2975 const SDLoc &dl = *JT.SL;
2976
2977 // Subtract the lowest switch case value from the value being switched on.
2978 SDValue SwitchOp = getValue(JTH.SValue);
2979 EVT VT = SwitchOp.getValueType();
2980 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2981 DAG.getConstant(JTH.First, dl, VT));
2982
2983 // The SDNode we just created, which holds the value being switched on minus
2984 // the smallest case value, needs to be copied to a virtual register so it
2985 // can be used as an index into the jump table in a subsequent basic block.
2986 // This value may be smaller or larger than the target's pointer type, and
2987 // therefore require extension or truncating.
2989 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2990
2991 unsigned JumpTableReg =
2993 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2994 JumpTableReg, SwitchOp);
2995 JT.Reg = JumpTableReg;
2996
2997 if (!JTH.FallthroughUnreachable) {
2998 // Emit the range check for the jump table, and branch to the default block
2999 // for the switch statement if the value being switched on exceeds the
3000 // largest case in the switch.
3001 SDValue CMP = DAG.getSetCC(
3003 Sub.getValueType()),
3004 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3005
3006 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3007 MVT::Other, CopyTo, CMP,
3008 DAG.getBasicBlock(JT.Default));
3009
3010 // Avoid emitting unnecessary branches to the next block.
3011 if (JT.MBB != NextBlock(SwitchBB))
3012 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3013 DAG.getBasicBlock(JT.MBB));
3014
3015 DAG.setRoot(BrCond);
3016 } else {
3017 // Avoid emitting unnecessary branches to the next block.
3018 if (JT.MBB != NextBlock(SwitchBB))
3019 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3020 DAG.getBasicBlock(JT.MBB)));
3021 else
3022 DAG.setRoot(CopyTo);
3023 }
3024}
3025
3026/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3027/// variable if there exists one.
3029 SDValue &Chain) {
3030 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3031 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3032 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3036 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3037 if (Global) {
3038 MachinePointerInfo MPInfo(Global);
3042 MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
3043 DAG.getEVTAlign(PtrTy));
3044 DAG.setNodeMemRefs(Node, {MemRef});
3045 }
3046 if (PtrTy != PtrMemTy)
3047 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3048 return SDValue(Node, 0);
3049}
3050
3051/// Codegen a new tail for a stack protector check ParentMBB which has had its
3052/// tail spliced into a stack protector check success bb.
3053///
3054/// For a high level explanation of how this fits into the stack protector
3055/// generation see the comment on the declaration of class
3056/// StackProtectorDescriptor.
3058 MachineBasicBlock *ParentBB) {
3059
3060 // First create the loads to the guard/stack slot for the comparison.
3062 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3063 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3064
3065 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3066 int FI = MFI.getStackProtectorIndex();
3067
3068 SDValue Guard;
3069 SDLoc dl = getCurSDLoc();
3070 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3071 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3072 Align Align =
3073 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3074
3075 // Generate code to load the content of the guard slot.
3076 SDValue GuardVal = DAG.getLoad(
3077 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3080
3081 if (TLI.useStackGuardXorFP())
3082 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3083
3084 // Retrieve guard check function, nullptr if instrumentation is inlined.
3085 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3086 // The target provides a guard check function to validate the guard value.
3087 // Generate a call to that function with the content of the guard slot as
3088 // argument.
3089 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3090 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3091
3094 Entry.Node = GuardVal;
3095 Entry.Ty = FnTy->getParamType(0);
3096 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3097 Entry.IsInReg = true;
3098 Args.push_back(Entry);
3099
3103 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3104 getValue(GuardCheckFn), std::move(Args));
3105
3106 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3107 DAG.setRoot(Result.second);
3108 return;
3109 }
3110
3111 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3112 // Otherwise, emit a volatile load to retrieve the stack guard value.
3113 SDValue Chain = DAG.getEntryNode();
3114 if (TLI.useLoadStackGuardNode()) {
3115 Guard = getLoadStackGuard(DAG, dl, Chain);
3116 } else {
3117 const Value *IRGuard = TLI.getSDagStackGuard(M);
3118 SDValue GuardPtr = getValue(IRGuard);
3119
3120 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3121 MachinePointerInfo(IRGuard, 0), Align,
3123 }
3124
3125 // Perform the comparison via a getsetcc.
3127 *DAG.getContext(),
3128 Guard.getValueType()),
3129 Guard, GuardVal, ISD::SETNE);
3130
3131 // If the guard/stackslot do not equal, branch to failure MBB.
3132 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3133 MVT::Other, GuardVal.getOperand(0),
3134 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3135 // Otherwise branch to success MBB.
3136 SDValue Br = DAG.getNode(ISD::BR, dl,
3137 MVT::Other, BrCond,
3139
3140 DAG.setRoot(Br);
3141}
3142
3143/// Codegen the failure basic block for a stack protector check.
3144///
3145/// A failure stack protector machine basic block consists simply of a call to
3146/// __stack_chk_fail().
3147///
3148/// For a high level explanation of how this fits into the stack protector
3149/// generation see the comment on the declaration of class
3150/// StackProtectorDescriptor.
3151void
3155 CallOptions.setDiscardResult(true);
3156 SDValue Chain =
3157 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3158 std::nullopt, CallOptions, getCurSDLoc())
3159 .second;
3160 // On PS4/PS5, the "return address" must still be within the calling
3161 // function, even if it's at the very end, so emit an explicit TRAP here.
3162 // Passing 'true' for doesNotReturn above won't generate the trap for us.
3163 if (TM.getTargetTriple().isPS())
3164 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3165 // WebAssembly needs an unreachable instruction after a non-returning call,
3166 // because the function return type can be different from __stack_chk_fail's
3167 // return type (void).
3168 if (TM.getTargetTriple().isWasm())
3169 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3170
3171 DAG.setRoot(Chain);
3172}
3173
3174/// visitBitTestHeader - This function emits necessary code to produce value
3175/// suitable for "bit tests"
3177 MachineBasicBlock *SwitchBB) {
3178 SDLoc dl = getCurSDLoc();
3179
3180 // Subtract the minimum value.
3181 SDValue SwitchOp = getValue(B.SValue);
3182 EVT VT = SwitchOp.getValueType();
3183 SDValue RangeSub =
3184 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3185
3186 // Determine the type of the test operands.
3188 bool UsePtrType = false;
3189 if (!TLI.isTypeLegal(VT)) {
3190 UsePtrType = true;
3191 } else {
3192 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3193 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3194 // Switch table case range are encoded into series of masks.
3195 // Just use pointer type, it's guaranteed to fit.
3196 UsePtrType = true;
3197 break;
3198 }
3199 }
3200 SDValue Sub = RangeSub;
3201 if (UsePtrType) {
3202 VT = TLI.getPointerTy(DAG.getDataLayout());
3203 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3204 }
3205
3206 B.RegVT = VT.getSimpleVT();
3207 B.Reg = FuncInfo.CreateReg(B.RegVT);
3208 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3209
3210 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3211
3212 if (!B.FallthroughUnreachable)
3213 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3214 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3215 SwitchBB->normalizeSuccProbs();
3216
3217 SDValue Root = CopyTo;
3218 if (!B.FallthroughUnreachable) {
3219 // Conditional branch to the default block.
3220 SDValue RangeCmp = DAG.getSetCC(dl,
3222 RangeSub.getValueType()),
3223 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3224 ISD::SETUGT);
3225
3226 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3227 DAG.getBasicBlock(B.Default));
3228 }
3229
3230 // Avoid emitting unnecessary branches to the next block.
3231 if (MBB != NextBlock(SwitchBB))
3232 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3233
3234 DAG.setRoot(Root);
3235}
3236
3237/// visitBitTestCase - this function produces one "bit test"
3239 MachineBasicBlock* NextMBB,
3240 BranchProbability BranchProbToNext,
3241 unsigned Reg,
3242 BitTestCase &B,
3243 MachineBasicBlock *SwitchBB) {
3244 SDLoc dl = getCurSDLoc();
3245 MVT VT = BB.RegVT;
3246 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3247 SDValue Cmp;
3248 unsigned PopCount = llvm::popcount(B.Mask);
3250 if (PopCount == 1) {
3251 // Testing for a single bit; just compare the shift count with what it
3252 // would need to be to shift a 1 bit in that position.
3253 Cmp = DAG.getSetCC(
3255 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3256 ISD::SETEQ);
3257 } else if (PopCount == BB.Range) {
3258 // There is only one zero bit in the range, test for it directly.
3259 Cmp = DAG.getSetCC(
3261 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3262 } else {
3263 // Make desired shift
3264 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3265 DAG.getConstant(1, dl, VT), ShiftOp);
3266
3267 // Emit bit tests and jumps
3268 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3269 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3270 Cmp = DAG.getSetCC(
3272 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3273 }
3274
3275 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3276 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3277 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3278 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3279 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3280 // one as they are relative probabilities (and thus work more like weights),
3281 // and hence we need to normalize them to let the sum of them become one.
3282 SwitchBB->normalizeSuccProbs();
3283
3284 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3285 MVT::Other, getControlRoot(),
3286 Cmp, DAG.getBasicBlock(B.TargetBB));
3287
3288 // Avoid emitting unnecessary branches to the next block.
3289 if (NextMBB != NextBlock(SwitchBB))
3290 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3291 DAG.getBasicBlock(NextMBB));
3292
3293 DAG.setRoot(BrAnd);
3294}
3295
3296void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3297 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3298
3299 // Retrieve successors. Look through artificial IR level blocks like
3300 // catchswitch for successors.
3301 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
3302 const BasicBlock *EHPadBB = I.getSuccessor(1);
3303 MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
3304
3305 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3306 // have to do anything here to lower funclet bundles.
3307 assert(!I.hasOperandBundlesOtherThan(
3308 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3309 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3310 LLVMContext::OB_cfguardtarget,
3311 LLVMContext::OB_clang_arc_attachedcall}) &&
3312 "Cannot lower invokes with arbitrary operand bundles yet!");
3313
3314 const Value *Callee(I.getCalledOperand());
3315 const Function *Fn = dyn_cast<Function>(Callee);
3316 if (isa<InlineAsm>(Callee))
3317 visitInlineAsm(I, EHPadBB);
3318 else if (Fn && Fn->isIntrinsic()) {
3319 switch (Fn->getIntrinsicID()) {
3320 default:
3321 llvm_unreachable("Cannot invoke this intrinsic");
3322 case Intrinsic::donothing:
3323 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3324 case Intrinsic::seh_try_begin:
3325 case Intrinsic::seh_scope_begin:
3326 case Intrinsic::seh_try_end:
3327 case Intrinsic::seh_scope_end:
3328 if (EHPadMBB)
3329 // a block referenced by EH table
3330 // so dtor-funclet not removed by opts
3331 EHPadMBB->setMachineBlockAddressTaken();
3332 break;
3333 case Intrinsic::experimental_patchpoint_void:
3334 case Intrinsic::experimental_patchpoint:
3335 visitPatchpoint(I, EHPadBB);
3336 break;
3337 case Intrinsic::experimental_gc_statepoint:
3338 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3339 break;
3340 case Intrinsic::wasm_rethrow: {
3341 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3342 // special because it can be invoked, so we manually lower it to a DAG
3343 // node here.
3345 Ops.push_back(getRoot()); // inchain
3347 Ops.push_back(
3348 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3350 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3352 break;
3353 }
3354 }
3355 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
3356 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3357 // Eventually we will support lowering the @llvm.experimental.deoptimize
3358 // intrinsic, and right now there are no plans to support other intrinsics
3359 // with deopt state.
3360 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3361 } else {
3362 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3363 }
3364
3365 // If the value of the invoke is used outside of its defining block, make it
3366 // available as a virtual register.
3367 // We already took care of the exported value for the statepoint instruction
3368 // during call to the LowerStatepoint.
3369 if (!isa<GCStatepointInst>(I)) {
3371 }
3372
3375 BranchProbability EHPadBBProb =
3376 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3378 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3379
3380 // Update successor info.
3381 addSuccessorWithProb(InvokeMBB, Return);
3382 for (auto &UnwindDest : UnwindDests) {
3383 UnwindDest.first->setIsEHPad();
3384 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3385 }
3386 InvokeMBB->normalizeSuccProbs();
3387
3388 // Drop into normal successor.
3390 DAG.getBasicBlock(Return)));
3391}
3392
3393void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3394 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3395
3396 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3397 // have to do anything here to lower funclet bundles.
3398 assert(!I.hasOperandBundlesOtherThan(
3399 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3400 "Cannot lower callbrs with arbitrary operand bundles yet!");
3401
3402 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3403 visitInlineAsm(I);
3405
3406 // Retrieve successors.
3408 Dests.insert(I.getDefaultDest());
3409 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3410
3411 // Update successor info.
3412 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3413 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3414 BasicBlock *Dest = I.getIndirectDest(i);
3416 Target->setIsInlineAsmBrIndirectTarget();
3417 Target->setMachineBlockAddressTaken();
3418 Target->setLabelMustBeEmitted();
3419 // Don't add duplicate machine successors.
3420 if (Dests.insert(Dest).second)
3421 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3422 }
3423 CallBrMBB->normalizeSuccProbs();
3424
3425 // Drop into default successor.
3427 MVT::Other, getControlRoot(),
3428 DAG.getBasicBlock(Return)));
3429}
3430
3431void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3432 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3433}
3434
3435void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3437 "Call to landingpad not in landing pad!");
3438
3439 // If there aren't registers to copy the values into (e.g., during SjLj
3440 // exceptions), then don't bother to create these DAG nodes.
3442 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3443 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3444 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3445 return;
3446
3447 // If landingpad's return type is token type, we don't create DAG nodes
3448 // for its exception pointer and selector value. The extraction of exception
3449 // pointer or selector value from token type landingpads is not currently
3450 // supported.
3451 if (LP.getType()->isTokenTy())
3452 return;
3453
3454 SmallVector<EVT, 2> ValueVTs;
3455 SDLoc dl = getCurSDLoc();
3456 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3457 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3458
3459 // Get the two live-in registers as SDValues. The physregs have already been
3460 // copied into virtual registers.
3461 SDValue Ops[2];
3463 Ops[0] = DAG.getZExtOrTrunc(
3467 dl, ValueVTs[0]);
3468 } else {
3469 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3470 }
3471 Ops[1] = DAG.getZExtOrTrunc(
3475 dl, ValueVTs[1]);
3476
3477 // Merge into one.
3479 DAG.getVTList(ValueVTs), Ops);
3480 setValue(&LP, Res);
3481}
3482
3485 // Update JTCases.
3486 for (JumpTableBlock &JTB : SL->JTCases)
3487 if (JTB.first.HeaderBB == First)
3488 JTB.first.HeaderBB = Last;
3489
3490 // Update BitTestCases.
3491 for (BitTestBlock &BTB : SL->BitTestCases)
3492 if (BTB.Parent == First)
3493 BTB.Parent = Last;
3494}
3495
3496void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3497 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3498
3499 // Update machine-CFG edges with unique successors.
3501 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3502 BasicBlock *BB = I.getSuccessor(i);
3503 bool Inserted = Done.insert(BB).second;
3504 if (!Inserted)
3505 continue;
3506
3507 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3508 addSuccessorWithProb(IndirectBrMBB, Succ);
3509 }
3510 IndirectBrMBB->normalizeSuccProbs();
3511
3513 MVT::Other, getControlRoot(),
3514 getValue(I.getAddress())));
3515}
3516
3517void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3519 return;
3520
3521 // We may be able to ignore unreachable behind a noreturn call.
3523 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode())) {
3524 if (Call->doesNotReturn())
3525 return;
3526 }
3527 }
3528
3529 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3530}
3531
3532void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3534 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3535 Flags.copyFMF(*FPOp);
3536
3537 SDValue Op = getValue(I.getOperand(0));
3538 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3539 Op, Flags);
3540 setValue(&I, UnNodeValue);
3541}
3542
3543void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3545 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3546 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3547 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3548 }
3549 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3550 Flags.setExact(ExactOp->isExact());
3551 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3552 Flags.setDisjoint(DisjointOp->isDisjoint());
3553 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3554 Flags.copyFMF(*FPOp);
3555
3556 SDValue Op1 = getValue(I.getOperand(0));
3557 SDValue Op2 = getValue(I.getOperand(1));
3558 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3559 Op1, Op2, Flags);
3560 setValue(&I, BinNodeValue);
3561}
3562
3563void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3564 SDValue Op1 = getValue(I.getOperand(0));
3565 SDValue Op2 = getValue(I.getOperand(1));
3566
3568 Op1.getValueType(), DAG.getDataLayout());
3569
3570 // Coerce the shift amount to the right type if we can. This exposes the
3571 // truncate or zext to optimization early.
3572 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3574 "Unexpected shift type");
3575 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3576 }
3577
3578 bool nuw = false;
3579 bool nsw = false;
3580 bool exact = false;
3581
3582 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3583
3584 if (const OverflowingBinaryOperator *OFBinOp =
3585 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3586 nuw = OFBinOp->hasNoUnsignedWrap();
3587 nsw = OFBinOp->hasNoSignedWrap();
3588 }
3589 if (const PossiblyExactOperator *ExactOp =
3590 dyn_cast<const PossiblyExactOperator>(&I))
3591 exact = ExactOp->isExact();
3592 }
3594 Flags.setExact(exact);
3595 Flags.setNoSignedWrap(nsw);
3596 Flags.setNoUnsignedWrap(nuw);
3597 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3598 Flags);
3599 setValue(&I, Res);
3600}
3601
3602void SelectionDAGBuilder::visitSDiv(const User &I) {
3603 SDValue Op1 = getValue(I.getOperand(0));
3604 SDValue Op2 = getValue(I.getOperand(1));
3605
3607 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3608 cast<PossiblyExactOperator>(&I)->isExact());
3610 Op2, Flags));
3611}
3612
3613void SelectionDAGBuilder::visitICmp(const User &I) {
3615 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3616 predicate = IC->getPredicate();
3617 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3618 predicate = ICmpInst::Predicate(IC->getPredicate());
3619 SDValue Op1 = getValue(I.getOperand(0));
3620 SDValue Op2 = getValue(I.getOperand(1));
3621 ISD::CondCode Opcode = getICmpCondCode(predicate);
3622
3623 auto &TLI = DAG.getTargetLoweringInfo();
3624 EVT MemVT =
3625 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3626
3627 // If a pointer's DAG type is larger than its memory type then the DAG values
3628 // are zero-extended. This breaks signed comparisons so truncate back to the
3629 // underlying type before doing the compare.
3630 if (Op1.getValueType() != MemVT) {
3631 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3632 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3633 }
3634
3636 I.getType());
3637 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3638}
3639
3640void SelectionDAGBuilder::visitFCmp(const User &I) {
3642 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3643 predicate = FC->getPredicate();
3644 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3645 predicate = FCmpInst::Predicate(FC->getPredicate());
3646 SDValue Op1 = getValue(I.getOperand(0));
3647 SDValue Op2 = getValue(I.getOperand(1));
3648
3649 ISD::CondCode Condition = getFCmpCondCode(predicate);
3650 auto *FPMO = cast<FPMathOperator>(&I);
3651 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3652 Condition = getFCmpCodeWithoutNaN(Condition);
3653
3655 Flags.copyFMF(*FPMO);
3656 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3657
3659 I.getType());
3660 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3661}
3662
3663// Check if the condition of the select has one use or two users that are both
3664// selects with the same condition.
3665static bool hasOnlySelectUsers(const Value *Cond) {
3666 return llvm::all_of(Cond->users(), [](const Value *V) {
3667 return isa<SelectInst>(V);
3668 });
3669}
3670
3671void SelectionDAGBuilder::visitSelect(const User &I) {
3672 SmallVector<EVT, 4> ValueVTs;
3674 ValueVTs);
3675 unsigned NumValues = ValueVTs.size();
3676 if (NumValues == 0) return;
3677
3678 SmallVector<SDValue, 4> Values(NumValues);
3679 SDValue Cond = getValue(I.getOperand(0));
3680 SDValue LHSVal = getValue(I.getOperand(1));
3681 SDValue RHSVal = getValue(I.getOperand(2));
3682 SmallVector<SDValue, 1> BaseOps(1, Cond);
3683 ISD::NodeType OpCode =
3684 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3685
3686 bool IsUnaryAbs = false;
3687 bool Negate = false;
3688
3690 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3691 Flags.copyFMF(*FPOp);
3692
3693 Flags.setUnpredictable(
3694 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3695
3696 // Min/max matching is only viable if all output VTs are the same.
3697 if (all_equal(ValueVTs)) {
3698 EVT VT = ValueVTs[0];
3699 LLVMContext &Ctx = *DAG.getContext();
3700 auto &TLI = DAG.getTargetLoweringInfo();
3701
3702 // We care about the legality of the operation after it has been type
3703 // legalized.
3704 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3705 VT = TLI.getTypeToTransformTo(Ctx, VT);
3706
3707 // If the vselect is legal, assume we want to leave this as a vector setcc +
3708 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3709 // min/max is legal on the scalar type.
3710 bool UseScalarMinMax = VT.isVector() &&
3712
3713 // ValueTracking's select pattern matching does not account for -0.0,
3714 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3715 // -0.0 is less than +0.0.
3716 Value *LHS, *RHS;
3717 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3719 switch (SPR.Flavor) {
3720 case SPF_UMAX: Opc = ISD::UMAX; break;
3721 case SPF_UMIN: Opc = ISD::UMIN; break;
3722 case SPF_SMAX: Opc = ISD::SMAX; break;
3723 case SPF_SMIN: Opc = ISD::SMIN; break;
3724 case SPF_FMINNUM:
3725 switch (SPR.NaNBehavior) {
3726 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3727 case SPNB_RETURNS_NAN: break;
3728 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3729 case SPNB_RETURNS_ANY:
3731 (UseScalarMinMax &&
3733 Opc = ISD::FMINNUM;
3734 break;
3735 }
3736 break;
3737 case SPF_FMAXNUM:
3738 switch (SPR.NaNBehavior) {
3739 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3740 case SPNB_RETURNS_NAN: break;
3741 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3742 case SPNB_RETURNS_ANY:
3744 (UseScalarMinMax &&
3746 Opc = ISD::FMAXNUM;
3747 break;
3748 }
3749 break;
3750 case SPF_NABS:
3751 Negate = true;
3752 [[fallthrough]];
3753 case SPF_ABS:
3754 IsUnaryAbs = true;
3755 Opc = ISD::ABS;
3756 break;
3757 default: break;
3758 }
3759
3760 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3761 (TLI.isOperationLegalOrCustomOrPromote(Opc, VT) ||
3762 (UseScalarMinMax &&
3763 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3764 // If the underlying comparison instruction is used by any other
3765 // instruction, the consumed instructions won't be destroyed, so it is
3766 // not profitable to convert to a min/max.
3767 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3768 OpCode = Opc;
3769 LHSVal = getValue(LHS);
3770 RHSVal = getValue(RHS);
3771 BaseOps.clear();
3772 }
3773
3774 if (IsUnaryAbs) {
3775 OpCode = Opc;
3776 LHSVal = getValue(LHS);
3777 BaseOps.clear();
3778 }
3779 }
3780
3781 if (IsUnaryAbs) {
3782 for (unsigned i = 0; i != NumValues; ++i) {
3783 SDLoc dl = getCurSDLoc();
3784 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3785 Values[i] =
3786 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3787 if (Negate)
3788 Values[i] = DAG.getNegative(Values[i], dl, VT);
3789 }
3790 } else {
3791 for (unsigned i = 0; i != NumValues; ++i) {
3792 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3793 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3794 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3795 Values[i] = DAG.getNode(
3796 OpCode, getCurSDLoc(),
3797 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3798 }
3799 }
3800
3802 DAG.getVTList(ValueVTs), Values));
3803}
3804
3805void SelectionDAGBuilder::visitTrunc(const User &I) {
3806 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3807 SDValue N = getValue(I.getOperand(0));
3809 I.getType());
3811}
3812
3813void SelectionDAGBuilder::visitZExt(const User &I) {
3814 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3815 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3816 SDValue N = getValue(I.getOperand(0));
3817 auto &TLI = DAG.getTargetLoweringInfo();
3818 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3819
3821 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3822 Flags.setNonNeg(PNI->hasNonNeg());
3823
3824 // Eagerly use nonneg information to canonicalize towards sign_extend if
3825 // that is the target's preference.
3826 // TODO: Let the target do this later.
3827 if (Flags.hasNonNeg() &&
3828 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3830 return;
3831 }
3832
3833 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3834}
3835
3836void SelectionDAGBuilder::visitSExt(const User &I) {
3837 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3838 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3839 SDValue N = getValue(I.getOperand(0));
3841 I.getType());
3843}
3844
3845void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3846 // FPTrunc is never a no-op cast, no need to check
3847 SDValue N = getValue(I.getOperand(0));
3848 SDLoc dl = getCurSDLoc();
3850 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3851 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3853 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3854}
3855
3856void SelectionDAGBuilder::visitFPExt(const User &I) {
3857 // FPExt is never a no-op cast, no need to check
3858 SDValue N = getValue(I.getOperand(0));
3860 I.getType());
3862}
3863
3864void SelectionDAGBuilder::visitFPToUI(const User &I) {
3865 // FPToUI is never a no-op cast, no need to check
3866 SDValue N = getValue(I.getOperand(0));
3868 I.getType());
3870}
3871
3872void SelectionDAGBuilder::visitFPToSI(const User &I) {
3873 // FPToSI is never a no-op cast, no need to check
3874 SDValue N = getValue(I.getOperand(0));
3876 I.getType());
3878}
3879
3880void SelectionDAGBuilder::visitUIToFP(const User &I) {
3881 // UIToFP is never a no-op cast, no need to check
3882 SDValue N = getValue(I.getOperand(0));
3884 I.getType());
3886}
3887
3888void SelectionDAGBuilder::visitSIToFP(const User &I) {
3889 // SIToFP is never a no-op cast, no need to check
3890 SDValue N = getValue(I.getOperand(0));
3892 I.getType());
3894}
3895
3896void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3897 // What to do depends on the size of the integer and the size of the pointer.
3898 // We can either truncate, zero extend, or no-op, accordingly.
3899 SDValue N = getValue(I.getOperand(0));
3900 auto &TLI = DAG.getTargetLoweringInfo();
3902 I.getType());
3903 EVT PtrMemVT =
3904 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3905 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3906 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3907 setValue(&I, N);
3908}
3909
3910void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3911 // What to do depends on the size of the integer and the size of the pointer.
3912 // We can either truncate, zero extend, or no-op, accordingly.
3913 SDValue N = getValue(I.getOperand(0));
3914 auto &TLI = DAG.getTargetLoweringInfo();
3915 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3916 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3917 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3918 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3919 setValue(&I, N);
3920}
3921
3922void SelectionDAGBuilder::visitBitCast(const User &I) {
3923 SDValue N = getValue(I.getOperand(0));
3924 SDLoc dl = getCurSDLoc();
3926 I.getType());
3927
3928 // BitCast assures us that source and destination are the same size so this is
3929 // either a BITCAST or a no-op.
3930 if (DestVT != N.getValueType())
3932 DestVT, N)); // convert types.
3933 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3934 // might fold any kind of constant expression to an integer constant and that
3935 // is not what we are looking for. Only recognize a bitcast of a genuine
3936 // constant integer as an opaque constant.
3937 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3938 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3939 /*isOpaque*/true));
3940 else
3941 setValue(&I, N); // noop cast.
3942}
3943
3944void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3946 const Value *SV = I.getOperand(0);
3947 SDValue N = getValue(SV);
3948 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3949
3950 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3951 unsigned DestAS = I.getType()->getPointerAddressSpace();
3952
3953 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3954 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3955
3956 setValue(&I, N);
3957}
3958
3959void SelectionDAGBuilder::visitInsertElement(const User &I) {
3961 SDValue InVec = getValue(I.getOperand(0));
3962 SDValue InVal = getValue(I.getOperand(1));
3963 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3966 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3967 InVec, InVal, InIdx));
3968}
3969
3970void SelectionDAGBuilder::visitExtractElement(const User &I) {
3972 SDValue InVec = getValue(I.getOperand(0));
3973 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3976 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3977 InVec, InIdx));
3978}
3979
3980void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3981 SDValue Src1 = getValue(I.getOperand(0));
3982 SDValue Src2 = getValue(I.getOperand(1));
3984 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3985 Mask = SVI->getShuffleMask();
3986 else
3987 Mask = cast<ConstantExpr>(I).getShuffleMask();
3988 SDLoc DL = getCurSDLoc();
3990 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3991 EVT SrcVT = Src1.getValueType();
3992
3993 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3994 VT.isScalableVector()) {
3995 // Canonical splat form of first element of first input vector.
3996 SDValue FirstElt =
3999 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4000 return;
4001 }
4002
4003 // For now, we only handle splats for scalable vectors.
4004 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4005 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4006 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4007
4008 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4009 unsigned MaskNumElts = Mask.size();
4010
4011 if (SrcNumElts == MaskNumElts) {
4012 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4013 return;
4014 }
4015
4016 // Normalize the shuffle vector since mask and vector length don't match.
4017 if (SrcNumElts < MaskNumElts) {
4018 // Mask is longer than the source vectors. We can use concatenate vector to
4019 // make the mask and vectors lengths match.
4020
4021 if (MaskNumElts % SrcNumElts == 0) {
4022 // Mask length is a multiple of the source vector length.
4023 // Check if the shuffle is some kind of concatenation of the input
4024 // vectors.
4025 unsigned NumConcat = MaskNumElts / SrcNumElts;
4026 bool IsConcat = true;
4027 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4028 for (unsigned i = 0; i != MaskNumElts; ++i) {
4029 int Idx = Mask[i];
4030 if (Idx < 0)
4031 continue;
4032 // Ensure the indices in each SrcVT sized piece are sequential and that
4033 // the same source is used for the whole piece.
4034 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4035 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4036 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4037 IsConcat = false;
4038 break;
4039 }
4040 // Remember which source this index came from.
4041 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4042 }
4043
4044 // The shuffle is concatenating multiple vectors together. Just emit
4045 // a CONCAT_VECTORS operation.
4046 if (IsConcat) {
4047 SmallVector<SDValue, 8> ConcatOps;
4048 for (auto Src : ConcatSrcs) {
4049 if (Src < 0)
4050 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4051 else if (Src == 0)
4052 ConcatOps.push_back(Src1);
4053 else
4054 ConcatOps.push_back(Src2);
4055 }
4056 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4057 return;
4058 }
4059 }
4060
4061 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4062 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4063 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4064 PaddedMaskNumElts);
4065
4066 // Pad both vectors with undefs to make them the same length as the mask.
4067 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4068
4069 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4070 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4071 MOps1[0] = Src1;
4072 MOps2[0] = Src2;
4073
4074 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4075 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4076
4077 // Readjust mask for new input vector length.
4078 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4079 for (unsigned i = 0; i != MaskNumElts; ++i) {
4080 int Idx = Mask[i];
4081 if (Idx >= (int)SrcNumElts)
4082 Idx -= SrcNumElts - PaddedMaskNumElts;
4083 MappedOps[i] = Idx;
4084 }
4085
4086 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4087
4088 // If the concatenated vector was padded, extract a subvector with the
4089 // correct number of elements.
4090 if (MaskNumElts != PaddedMaskNumElts)
4093
4094 setValue(&I, Result);
4095 return;
4096 }
4097
4098 if (SrcNumElts > MaskNumElts) {
4099 // Analyze the access pattern of the vector to see if we can extract
4100 // two subvectors and do the shuffle.
4101 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
4102 bool CanExtract = true;
4103 for (int Idx : Mask) {
4104 unsigned Input = 0;
4105 if (Idx < 0)
4106 continue;
4107
4108 if (Idx >= (int)SrcNumElts) {
4109 Input = 1;
4110 Idx -= SrcNumElts;
4111 }
4112
4113 // If all the indices come from the same MaskNumElts sized portion of
4114 // the sources we can use extract. Also make sure the extract wouldn't
4115 // extract past the end of the source.
4116 int NewStartIdx = alignDown(Idx, MaskNumElts);
4117 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4118 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4119 CanExtract = false;
4120 // Make sure we always update StartIdx as we use it to track if all
4121 // elements are undef.
4122 StartIdx[Input] = NewStartIdx;
4123 }
4124
4125 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4126 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4127 return;
4128 }
4129 if (CanExtract) {
4130 // Extract appropriate subvector and generate a vector shuffle
4131 for (unsigned Input = 0; Input < 2; ++Input) {
4132 SDValue &Src = Input == 0 ? Src1 : Src2;
4133 if (StartIdx[Input] < 0)
4134 Src = DAG.getUNDEF(VT);
4135 else {
4136 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4137 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4138 }
4139 }
4140
4141 // Calculate new mask.
4142 SmallVector<int, 8> MappedOps(Mask);
4143 for (int &Idx : MappedOps) {
4144 if (Idx >= (int)SrcNumElts)
4145 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4146 else if (Idx >= 0)
4147 Idx -= StartIdx[0];
4148 }
4149
4150 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4151 return;
4152 }
4153 }
4154
4155 // We can't use either concat vectors or extract subvectors so fall back to
4156 // replacing the shuffle with extract and build vector.
4157 // to insert and build vector.
4158 EVT EltVT = VT.getVectorElementType();
4160 for (int Idx : Mask) {
4161 SDValue Res;
4162
4163 if (Idx < 0) {
4164 Res = DAG.getUNDEF(EltVT);
4165 } else {
4166 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4167 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4168
4169 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4171 }
4172
4173 Ops.push_back(Res);
4174 }
4175
4176 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4177}
4178
4179void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4180 ArrayRef<unsigned> Indices = I.getIndices();
4181 const Value *Op0 = I.getOperand(0);
4182 const Value *Op1 = I.getOperand(1);
4183 Type *AggTy = I.getType();
4184 Type *ValTy = Op1->getType();
4185 bool IntoUndef = isa<UndefValue>(Op0);
4186 bool FromUndef = isa<UndefValue>(Op1);
4187
4188 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4189
4191 SmallVector<EVT, 4> AggValueVTs;
4192 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4193 SmallVector<EVT, 4> ValValueVTs;
4194 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4195
4196 unsigned NumAggValues = AggValueVTs.size();
4197 unsigned NumValValues = ValValueVTs.size();
4198 SmallVector<SDValue, 4> Values(NumAggValues);
4199
4200 // Ignore an insertvalue that produces an empty object
4201 if (!NumAggValues) {
4202 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4203 return;
4204 }
4205
4206 SDValue Agg = getValue(Op0);
4207 unsigned i = 0;
4208 // Copy the beginning value(s) from the original aggregate.
4209 for (; i != LinearIndex; ++i)
4210 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4211 SDValue(Agg.getNode(), Agg.getResNo() + i);
4212 // Copy values from the inserted value(s).
4213 if (NumValValues) {
4214 SDValue Val = getValue(Op1);
4215 for (; i != LinearIndex + NumValValues; ++i)
4216 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4217 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4218 }
4219 // Copy remaining value(s) from the original aggregate.
4220 for (; i != NumAggValues; ++i)
4221 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4222 SDValue(Agg.getNode(), Agg.getResNo() + i);
4223
4225 DAG.getVTList(AggValueVTs), Values));
4226}
4227
4228void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4229 ArrayRef<unsigned> Indices = I.getIndices();
4230 const Value *Op0 = I.getOperand(0);
4231 Type *AggTy = Op0->getType();
4232 Type *ValTy = I.getType();
4233 bool OutOfUndef = isa<UndefValue>(Op0);
4234
4235 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4236
4238 SmallVector<EVT, 4> ValValueVTs;
4239 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4240
4241 unsigned NumValValues = ValValueVTs.size();
4242
4243 // Ignore a extractvalue that produces an empty object
4244 if (!NumValValues) {
4245 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4246 return;
4247 }
4248
4249 SmallVector<SDValue, 4> Values(NumValValues);
4250
4251 SDValue Agg = getValue(Op0);
4252 // Copy out the selected value(s).
4253 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4254 Values[i - LinearIndex] =
4255 OutOfUndef ?
4256 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4257 SDValue(Agg.getNode(), Agg.getResNo() + i);
4258
4260 DAG.getVTList(ValValueVTs), Values));
4261}
4262
4263void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4264 Value *Op0 = I.getOperand(0);
4265 // Note that the pointer operand may be a vector of pointers. Take the scalar
4266 // element which holds a pointer.
4267 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4268 SDValue N = getValue(Op0);
4269 SDLoc dl = getCurSDLoc();
4270 auto &TLI = DAG.getTargetLoweringInfo();
4271
4272 // Normalize Vector GEP - all scalar operands should be converted to the
4273 // splat vector.
4274 bool IsVectorGEP = I.getType()->isVectorTy();
4275 ElementCount VectorElementCount =
4276 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4278
4279 if (IsVectorGEP && !N.getValueType().isVector()) {
4281 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
4282 N = DAG.getSplat(VT, dl, N);
4283 }
4284
4285 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
4286 GTI != E; ++GTI) {
4287 const Value *Idx = GTI.getOperand();
4288 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4289 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4290 if (Field) {
4291 // N = N + Offset
4294
4295 // In an inbounds GEP with an offset that is nonnegative even when
4296 // interpreted as signed, assume there is no unsigned overflow.
4298 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
4299 Flags.setNoUnsignedWrap(true);
4300
4301 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
4302 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
4303 }
4304 } else {
4305 // IdxSize is the width of the arithmetic according to IR semantics.
4306 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4307 // (and fix up the result later).
4308 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4309 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4310 TypeSize ElementSize =
4311 GTI.getSequentialElementStride(DAG.getDataLayout());
4312 // We intentionally mask away the high bits here; ElementSize may not
4313 // fit in IdxTy.
4314 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
4315 bool ElementScalable = ElementSize.isScalable();
4316
4317 // If this is a scalar constant or a splat vector of constants,
4318 // handle it quickly.
4319 const auto *C = dyn_cast<Constant>(Idx);
4320 if (C && isa<VectorType>(C->getType()))
4321 C = C->getSplatValue();
4322
4323 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4324 if (CI && CI->isZero())
4325 continue;
4326 if (CI && !ElementScalable) {
4327 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4329 SDValue OffsVal;
4330 if (IsVectorGEP)
4331 OffsVal = DAG.getConstant(
4332 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4333 else
4334 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4335
4336 // In an inbounds GEP with an offset that is nonnegative even when
4337 // interpreted as signed, assume there is no unsigned overflow.
4339 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
4340 Flags.setNoUnsignedWrap(true);
4341
4342 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4343
4344 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4345 continue;
4346 }
4347
4348 // N = N + Idx * ElementMul;
4349 SDValue IdxN = getValue(Idx);
4350
4351 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4353 VectorElementCount);
4354 IdxN = DAG.getSplat(VT, dl, IdxN);
4355 }
4356
4357 // If the index is smaller or larger than intptr_t, truncate or extend
4358 // it.
4359 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4360
4361 if (ElementScalable) {
4362 EVT VScaleTy = N.getValueType().getScalarType();
4363 SDValue VScale = DAG.getNode(
4364 ISD::VSCALE, dl, VScaleTy,
4365 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4366 if (IsVectorGEP)
4367 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4368 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4369 } else {
4370 // If this is a multiply by a power of two, turn it into a shl
4371 // immediately. This is a very common case.
4372 if (ElementMul != 1) {
4373 if (ElementMul.isPowerOf2()) {
4374 unsigned Amt = ElementMul.logBase2();
4375 IdxN = DAG.getNode(ISD::SHL, dl,
4376 N.getValueType(), IdxN,
4377 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4378 } else {
4379 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4380 IdxN.getValueType());
4381 IdxN = DAG.getNode(ISD::MUL, dl,
4382 N.getValueType(), IdxN, Scale);
4383 }
4384 }
4385 }
4386
4387 N = DAG.getNode(ISD::ADD, dl,
4388 N.getValueType(), N, IdxN);
4389 }
4390 }
4391
4392 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4393 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4394 if (IsVectorGEP) {
4395 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4396 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4397 }
4398
4399 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4400 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4401
4402 setValue(&I, N);
4403}
4404
4405void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4406 // If this is a fixed sized alloca in the entry block of the function,
4407 // allocate it statically on the stack.
4408 if (FuncInfo.StaticAllocaMap.count(&I))
4409 return; // getValue will auto-populate this.
4410
4411 SDLoc dl = getCurSDLoc();
4412 Type *Ty = I.getAllocatedType();
4414 auto &DL = DAG.getDataLayout();
4415 TypeSize TySize = DL.getTypeAllocSize(Ty);
4416 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4417
4418 SDValue AllocSize = getValue(I.getArraySize());
4419
4420 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4421 if (AllocSize.getValueType() != IntPtr)
4422 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4423
4424 if (TySize.isScalable())
4425 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4426 DAG.getVScale(dl, IntPtr,
4427 APInt(IntPtr.getScalarSizeInBits(),
4428 TySize.getKnownMinValue())));
4429 else {
4430 SDValue TySizeValue =
4432 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4433 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4434 }
4435
4436 // Handle alignment. If the requested alignment is less than or equal to
4437 // the stack alignment, ignore it. If the size is greater than or equal to
4438 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4440 if (*Alignment <= StackAlign)
4441 Alignment = std::nullopt;
4442
4443 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4444 // Round the size of the allocation up to the stack alignment size
4445 // by add SA-1 to the size. This doesn't overflow because we're computing
4446 // an address inside an alloca.
4448 Flags.setNoUnsignedWrap(true);
4449 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4450 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4451
4452 // Mask out the low bits for alignment purposes.
4453 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4454 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4455
4456 SDValue Ops[] = {
4457 getRoot(), AllocSize,
4458 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4459 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4460 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4461 setValue(&I, DSA);
4462 DAG.setRoot(DSA.getValue(1));
4463
4465}
4466
4467static const MDNode *getRangeMetadata(const Instruction &I) {
4468 // If !noundef is not present, then !range violation results in a poison
4469 // value rather than immediate undefined behavior. In theory, transferring
4470 // these annotations to SDAG is fine, but in practice there are key SDAG
4471 // transforms that are known not to be poison-safe, such as folding logical
4472 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4473 // also present.
4474 if (!I.hasMetadata(LLVMContext::MD_noundef))
4475 return nullptr;
4476 return I.getMetadata(LLVMContext::MD_range);
4477}
4478
4479void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4480 if (I.isAtomic())
4481 return visitAtomicLoad(I);
4482
4484 const Value *SV = I.getOperand(0);
4485 if (TLI.supportSwiftError()) {
4486 // Swifterror values can come from either a function parameter with
4487 // swifterror attribute or an alloca with swifterror attribute.
4488 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4489 if (Arg->hasSwiftErrorAttr())
4490 return visitLoadFromSwiftError(I);
4491 }
4492
4493 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4494 if (Alloca->isSwiftError())
4495 return visitLoadFromSwiftError(I);
4496 }
4497 }
4498
4499 SDValue Ptr = getValue(SV);
4500
4501 Type *Ty = I.getType();
4502 SmallVector<EVT, 4> ValueVTs, MemVTs;
4504 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4505 unsigned NumValues = ValueVTs.size();
4506 if (NumValues == 0)
4507 return;
4508
4509 Align Alignment = I.getAlign();
4510 AAMDNodes AAInfo = I.getAAMetadata();
4511 const MDNode *Ranges = getRangeMetadata(I);
4512 bool isVolatile = I.isVolatile();
4513 MachineMemOperand::Flags MMOFlags =
4515
4516 SDValue Root;
4517 bool ConstantMemory = false;
4518 if (isVolatile)
4519 // Serialize volatile loads with other side effects.
4520 Root = getRoot();
4521 else if (NumValues > MaxParallelChains)
4522 Root = getMemoryRoot();
4523 else if (AA &&
4525 SV,
4527 AAInfo))) {
4528 // Do not serialize (non-volatile) loads of constant memory with anything.
4529 Root = DAG.getEntryNode();
4530 ConstantMemory = true;
4532 } else {
4533 // Do not serialize non-volatile loads against each other.
4534 Root = DAG.getRoot();
4535 }
4536
4537 SDLoc dl = getCurSDLoc();
4538
4539 if (isVolatile)
4540 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4541
4542 SmallVector<SDValue, 4> Values(NumValues);
4543 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4544
4545 unsigned ChainI = 0;
4546 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4547 // Serializing loads here may result in excessive register pressure, and
4548 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4549 // could recover a bit by hoisting nodes upward in the chain by recognizing
4550 // they are side-effect free or do not alias. The optimizer should really
4551 // avoid this case by converting large object/array copies to llvm.memcpy
4552 // (MaxParallelChains should always remain as failsafe).
4553 if (ChainI == MaxParallelChains) {
4554 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4555 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4556 ArrayRef(Chains.data(), ChainI));
4557 Root = Chain;
4558 ChainI = 0;
4559 }
4560
4561 // TODO: MachinePointerInfo only supports a fixed length offset.
4562 MachinePointerInfo PtrInfo =
4563 !Offsets[i].isScalable() || Offsets[i].isZero()
4564 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4566
4567 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4568 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4569 MMOFlags, AAInfo, Ranges);
4570 Chains[ChainI] = L.getValue(1);
4571
4572 if (MemVTs[i] != ValueVTs[i])
4573 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4574
4575 Values[i] = L;
4576 }
4577
4578 if (!ConstantMemory) {
4579 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4580 ArrayRef(Chains.data(), ChainI));
4581 if (isVolatile)
4582 DAG.setRoot(Chain);
4583 else
4584 PendingLoads.push_back(Chain);
4585 }
4586
4588 DAG.getVTList(ValueVTs), Values));
4589}
4590
4591void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4593 "call visitStoreToSwiftError when backend supports swifterror");
4594
4595 SmallVector<EVT, 4> ValueVTs;
4597 const Value *SrcV = I.getOperand(0);
4599 SrcV->getType(), ValueVTs, &Offsets, 0);
4600 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4601 "expect a single EVT for swifterror");
4602
4603 SDValue Src = getValue(SrcV);
4604 // Create a virtual register, then update the virtual register.
4605 Register VReg =
4606 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4607 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4608 // Chain can be getRoot or getControlRoot.
4609 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4610 SDValue(Src.getNode(), Src.getResNo()));
4611 DAG.setRoot(CopyNode);
4612}
4613
4614void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4616 "call visitLoadFromSwiftError when backend supports swifterror");
4617
4618 assert(!I.isVolatile() &&
4619 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4620 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4621 "Support volatile, non temporal, invariant for load_from_swift_error");
4622
4623 const Value *SV = I.getOperand(0);
4624 Type *Ty = I.getType();
4625 assert(
4626 (!AA ||
4629 I.getAAMetadata()))) &&
4630 "load_from_swift_error should not be constant memory");
4631
4632 SmallVector<EVT, 4> ValueVTs;
4635 ValueVTs, &Offsets, 0);
4636 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4637 "expect a single EVT for swifterror");
4638
4639 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4641 getRoot(), getCurSDLoc(),
4642 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4643
4644 setValue(&I, L);
4645}
4646
4647void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4648 if (I.isAtomic())
4649 return visitAtomicStore(I);
4650
4651 const Value *SrcV = I.getOperand(0);
4652 const Value *PtrV = I.getOperand(1);
4653
4655 if (TLI.supportSwiftError()) {
4656 // Swifterror values can come from either a function parameter with
4657 // swifterror attribute or an alloca with swifterror attribute.
4658 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4659 if (Arg->hasSwiftErrorAttr())
4660 return visitStoreToSwiftError(I);
4661 }
4662
4663 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4664 if (Alloca->isSwiftError())
4665 return visitStoreToSwiftError(I);
4666 }
4667 }
4668
4669 SmallVector<EVT, 4> ValueVTs, MemVTs;
4672 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4673 unsigned NumValues = ValueVTs.size();
4674 if (NumValues == 0)
4675 return;
4676
4677 // Get the lowered operands. Note that we do this after
4678 // checking if NumResults is zero, because with zero results
4679 // the operands won't have values in the map.
4680 SDValue Src = getValue(SrcV);
4681 SDValue Ptr = getValue(PtrV);
4682
4683 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4684 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4685 SDLoc dl = getCurSDLoc();
4686 Align Alignment = I.getAlign();
4687 AAMDNodes AAInfo = I.getAAMetadata();
4688
4689 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4690
4691 unsigned ChainI = 0;
4692 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4693 // See visitLoad comments.
4694 if (ChainI == MaxParallelChains) {
4695 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4696 ArrayRef(Chains.data(), ChainI));
4697 Root = Chain;
4698 ChainI = 0;
4699 }
4700
4701 // TODO: MachinePointerInfo only supports a fixed length offset.
4702 MachinePointerInfo PtrInfo =
4703 !Offsets[i].isScalable() || Offsets[i].isZero()
4704 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4706
4707 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4708 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4709 if (MemVTs[i] != ValueVTs[i])
4710 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4711 SDValue St =
4712 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4713 Chains[ChainI] = St;
4714 }
4715
4716 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4717 ArrayRef(Chains.data(), ChainI));
4718 setValue(&I, StoreNode);
4719 DAG.setRoot(StoreNode);
4720}
4721
4722void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4723 bool IsCompressing) {
4724 SDLoc sdl = getCurSDLoc();
4725
4726 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4727 Align &Alignment) {
4728 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4729 Src0 = I.getArgOperand(0);
4730 Ptr = I.getArgOperand(1);
4731 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getAlignValue();
4732 Mask = I.getArgOperand(3);
4733 };
4734 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4735 Align &Alignment) {
4736 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4737 Src0 = I.getArgOperand(0);
4738 Ptr = I.getArgOperand(1);
4739 Mask = I.getArgOperand(2);
4740 Alignment = I.getParamAlign(1).valueOrOne();
4741 };
4742
4743 Value *PtrOperand, *MaskOperand, *Src0Operand;
4744 Align Alignment;
4745 if (IsCompressing)
4746 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4747 else
4748 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4749
4750 SDValue Ptr = getValue(PtrOperand);
4751 SDValue Src0 = getValue(Src0Operand);
4752 SDValue Mask = getValue(MaskOperand);
4753 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4754
4755 EVT VT = Src0.getValueType();
4756
4759 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4760 SDValue StoreNode =
4761 DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4762 ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4763 DAG.setRoot(StoreNode);
4764 setValue(&I, StoreNode);
4765}
4766
4767// Get a uniform base for the Gather/Scatter intrinsic.
4768// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4769// We try to represent it as a base pointer + vector of indices.
4770// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4771// The first operand of the GEP may be a single pointer or a vector of pointers
4772// Example:
4773// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4774// or
4775// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4776// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4777//
4778// When the first GEP operand is a single pointer - it is the uniform base we
4779// are looking for. If first operand of the GEP is a splat vector - we
4780// extract the splat value and use it as a uniform base.
4781// In all other cases the function returns 'false'.
4783 ISD::MemIndexType &IndexType, SDValue &Scale,
4784 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4785 uint64_t ElemSize) {
4786 SelectionDAG& DAG = SDB->DAG;
4787 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4788 const DataLayout &DL = DAG.getDataLayout();
4789
4790 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4791
4792 // Handle splat constant pointer.
4793 if (auto *C = dyn_cast<Constant>(Ptr)) {
4794 C = C->getSplatValue();
4795 if (!C)
4796 return false;
4797
4798 Base = SDB->getValue(C);
4799
4800 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4801 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4802 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4803 IndexType = ISD::SIGNED_SCALED;
4804 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4805 return true;
4806 }
4807
4808 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4809 if (!GEP || GEP->getParent() != CurBB)
4810 return false;
4811
4812 if (GEP->getNumOperands() != 2)
4813 return false;
4814
4815 const Value *BasePtr = GEP->getPointerOperand();
4816 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4817
4818 // Make sure the base is scalar and the index is a vector.
4819 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4820 return false;
4821
4822 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4823 if (ScaleVal.isScalable())
4824 return false;
4825
4826 // Target may not support the required addressing mode.
4827 if (ScaleVal != 1 &&
4828 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4829 return false;
4830
4831 Base = SDB->getValue(BasePtr);
4832 Index = SDB->getValue(IndexVal);
4833 IndexType = ISD::SIGNED_SCALED;
4834
4835 Scale =
4836 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4837 return true;
4838}
4839
4840void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4841 SDLoc sdl = getCurSDLoc();
4842
4843 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4844 const Value *Ptr = I.getArgOperand(1);
4845 SDValue Src0 = getValue(I.getArgOperand(0));
4846 SDValue Mask = getValue(I.getArgOperand(3));
4847 EVT VT = Src0.getValueType();
4848 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4849 ->getMaybeAlignValue()
4850 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4852
4853 SDValue Base;
4854 SDValue Index;
4855 ISD::MemIndexType IndexType;
4856 SDValue Scale;
4857 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4858 I.getParent(), VT.getScalarStoreSize());
4859
4860 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4863 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4864 if (!UniformBase) {
4866 Index = getValue(Ptr);
4867 IndexType = ISD::SIGNED_SCALED;
4868 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4869 }
4870
4871 EVT IdxVT = Index.getValueType();
4872 EVT EltTy = IdxVT.getVectorElementType();
4873 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4874 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4875 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4876 }
4877
4878 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4879 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4880 Ops, MMO, IndexType, false);
4881 DAG.setRoot(Scatter);
4882 setValue(&I, Scatter);
4883}
4884
4885void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4886 SDLoc sdl = getCurSDLoc();
4887
4888 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4889 Align &Alignment) {
4890 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4891 Ptr = I.getArgOperand(0);
4892 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getAlignValue();
4893 Mask = I.getArgOperand(2);
4894 Src0 = I.getArgOperand(3);
4895 };
4896 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4897 Align &Alignment) {
4898 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4899 Ptr = I.getArgOperand(0);
4900 Alignment = I.getParamAlign(0).valueOrOne();
4901 Mask = I.getArgOperand(1);
4902 Src0 = I.getArgOperand(2);
4903 };
4904
4905 Value *PtrOperand, *MaskOperand, *Src0Operand;
4906 Align Alignment;
4907 if (IsExpanding)
4908 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4909 else
4910 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4911
4912 SDValue Ptr = getValue(PtrOperand);
4913 SDValue Src0 = getValue(Src0Operand);
4914 SDValue Mask = getValue(MaskOperand);
4915 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4916
4917 EVT VT = Src0.getValueType();
4918 AAMDNodes AAInfo = I.getAAMetadata();
4919 const MDNode *Ranges = getRangeMetadata(I);
4920
4921 // Do not serialize masked loads of constant memory with anything.
4922 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4923 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4924
4925 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4926
4929 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
4930
4931 SDValue Load =
4932 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4933 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4934 if (AddToChain)
4935 PendingLoads.push_back(Load.getValue(1));
4936 setValue(&I, Load);
4937}
4938
4939void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4940 SDLoc sdl = getCurSDLoc();
4941
4942 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4943 const Value *Ptr = I.getArgOperand(0);
4944 SDValue Src0 = getValue(I.getArgOperand(3));
4945 SDValue Mask = getValue(I.getArgOperand(2));
4946
4948 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4949 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4950 ->getMaybeAlignValue()
4951 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4952
4953 const MDNode *Ranges = getRangeMetadata(I);
4954
4955 SDValue Root = DAG.getRoot();
4956 SDValue Base;
4957 SDValue Index;
4958 ISD::MemIndexType IndexType;
4959 SDValue Scale;
4960 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4961 I.getParent(), VT.getScalarStoreSize());
4962 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4965 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
4966 Ranges);
4967
4968 if (!UniformBase) {
4970 Index = getValue(Ptr);
4971 IndexType = ISD::SIGNED_SCALED;
4972 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4973 }
4974
4975 EVT IdxVT = Index.getValueType();
4976 EVT EltTy = IdxVT.getVectorElementType();
4977 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4978 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4979 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4980 }
4981
4982 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4983 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4984 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4985
4986 PendingLoads.push_back(Gather.getValue(1));
4987 setValue(&I, Gather);
4988}
4989
4990void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4991 SDLoc dl = getCurSDLoc();
4992 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4993 AtomicOrdering FailureOrdering = I.getFailureOrdering();
4994 SyncScope::ID SSID = I.getSyncScopeID();
4995
4996 SDValue InChain = getRoot();
4997
4998 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4999 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5000
5003
5006 MachinePointerInfo(I.getPointerOperand()), Flags,
5008 AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering);
5009
5011 dl, MemVT, VTs, InChain,
5012 getValue(I.getPointerOperand()),
5013 getValue(I.getCompareOperand()),
5014 getValue(I.getNewValOperand()), MMO);
5015
5016 SDValue OutChain = L.getValue(2);
5017
5018 setValue(&I, L);
5019 DAG.setRoot(OutChain);
5020}
5021
5022void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5023 SDLoc dl = getCurSDLoc();
5025 switch (I.getOperation()) {
5026 default: llvm_unreachable("Unknown atomicrmw operation");
5044 break;
5047 break;
5048 }
5049 AtomicOrdering Ordering = I.getOrdering();
5050 SyncScope::ID SSID = I.getSyncScopeID();
5051
5052 SDValue InChain = getRoot();
5053
5054 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5057
5060 MachinePointerInfo(I.getPointerOperand()), Flags,
5062 AAMDNodes(), nullptr, SSID, Ordering);
5063
5064 SDValue L =
5065 DAG.getAtomic(NT, dl, MemVT, InChain,
5066 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5067 MMO);
5068
5069 SDValue OutChain = L.getValue(1);
5070
5071 setValue(&I, L);
5072 DAG.setRoot(OutChain);
5073}
5074
5075void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5076 SDLoc dl = getCurSDLoc();
5078 SDValue Ops[3];
5079 Ops[0] = getRoot();
5080 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5082 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5084 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5085 setValue(&I, N);
5086 DAG.setRoot(N);
5087}
5088
5089void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5090 SDLoc dl = getCurSDLoc();
5091 AtomicOrdering Order = I.getOrdering();
5092 SyncScope::ID SSID = I.getSyncScopeID();
5093
5094 SDValue InChain = getRoot();
5095
5097 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5098 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5099
5100 if (!TLI.supportsUnalignedAtomics() &&
5101 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5102 report_fatal_error("Cannot generate unaligned atomic load");
5103
5105
5107 MachinePointerInfo(I.getPointerOperand()), Flags,
5108 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5109 nullptr, SSID, Order);
5110
5111 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5112
5113 SDValue Ptr = getValue(I.getPointerOperand());
5114 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
5115 Ptr, MMO);
5116
5117 SDValue OutChain = L.getValue(1);
5118 if (MemVT != VT)
5119 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5120
5121 setValue(&I, L);
5122 DAG.setRoot(OutChain);
5123}
5124
5125void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5126 SDLoc dl = getCurSDLoc();
5127
5128 AtomicOrdering Ordering = I.getOrdering();
5129 SyncScope::ID SSID = I.getSyncScopeID();
5130
5131 SDValue InChain = getRoot();
5132
5134 EVT MemVT =
5135 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5136
5137 if (!TLI.supportsUnalignedAtomics() &&
5138 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5139 report_fatal_error("Cannot generate unaligned atomic store");
5140
5142
5145 MachinePointerInfo(I.getPointerOperand()), Flags,
5146 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5147 nullptr, SSID, Ordering);
5148
5149 SDValue Val = getValue(I.getValueOperand());
5150 if (Val.getValueType() != MemVT)
5151 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5152 SDValue Ptr = getValue(I.getPointerOperand());
5153
5154 SDValue OutChain =
5155 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5156
5157 setValue(&I, OutChain);
5158 DAG.setRoot(OutChain);
5159}
5160
5161/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5162/// node.
5163void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5164 unsigned Intrinsic) {
5165 // Ignore the callsite's attributes. A specific call site may be marked with
5166 // readnone, but the lowering code will expect the chain based on the
5167 // definition.
5168 const Function *F = I.getCalledFunction();
5169 bool HasChain = !F->doesNotAccessMemory();
5170 bool OnlyLoad = HasChain && F->onlyReadsMemory();
5171
5172 // Build the operand list.
5174 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5175 if (OnlyLoad) {
5176 // We don't need to serialize loads against other loads.
5177 Ops.push_back(DAG.getRoot());
5178 } else {
5179 Ops.push_back(getRoot());
5180 }
5181 }
5182
5183 // Info is set by getTgtMemIntrinsic
5186 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5188 Intrinsic);
5189
5190 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5191 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5193 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5195
5196 // Add all operands of the call to the operand list.
5197 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5198 const Value *Arg = I.getArgOperand(i);
5199 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5200 Ops.push_back(getValue(Arg));
5201 continue;
5202 }
5203
5204 // Use TargetConstant instead of a regular constant for immarg.
5205 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5206 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5207 assert(CI->getBitWidth() <= 64 &&
5208 "large intrinsic immediates not handled");
5209 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5210 } else {
5211 Ops.push_back(
5212 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5213 }
5214 }
5215
5216 SmallVector<EVT, 4> ValueVTs;
5217 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5218
5219 if (HasChain)
5220 ValueVTs.push_back(MVT::Other);
5221
5222 SDVTList VTs = DAG.getVTList(ValueVTs);
5223
5224 // Propagate fast-math-flags from IR to node(s).
5226 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5227 Flags.copyFMF(*FPMO);
5228 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5229
5230 // Create the node.
5232
5233 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5234 auto *Token = Bundle->Inputs[0].get();
5235 SDValue ConvControlToken = getValue(Token);
5236 assert(Ops.back().getValueType() != MVT::Glue &&
5237 "Did not expected another glue node here.");
5238 ConvControlToken =
5239 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5240 Ops.push_back(ConvControlToken);
5241 }
5242
5243 // In some cases, custom collection of operands from CallInst I may be needed.
5245 if (IsTgtIntrinsic) {
5246 // This is target intrinsic that touches memory
5247 //
5248 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5249 // didn't yield anything useful.
5251 if (Info.ptrVal)
5252 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5253 else if (Info.fallbackAddressSpace)
5254 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5255 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
5256 Info.memVT, MPI, Info.align, Info.flags,
5257 Info.size, I.getAAMetadata());
5258 } else if (!HasChain) {
5260 } else if (!I.getType()->isVoidTy()) {
5262 } else {
5264 }
5265
5266 if (HasChain) {
5267 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5268 if (OnlyLoad)
5269 PendingLoads.push_back(Chain);
5270 else
5271 DAG.setRoot(Chain);
5272 }
5273
5274 if (!I.getType()->isVoidTy()) {
5275 if (!isa<VectorType>(I.getType()))
5276 Result = lowerRangeToAssertZExt(DAG, I, Result);
5277
5278 MaybeAlign Alignment = I.getRetAlign();
5279
5280 // Insert `assertalign` node if there's an alignment.
5281 if (InsertAssertAlign && Alignment) {
5282 Result =
5283 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5284 }
5285
5286 setValue(&I, Result);
5287 }
5288}
5289
5290/// GetSignificand - Get the significand and build it into a floating-point
5291/// number with exponent of 1:
5292///
5293/// Op = (Op & 0x007fffff) | 0x3f800000;
5294///
5295/// where Op is the hexadecimal representation of floating point value.
5297 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5298 DAG.getConstant(0x007fffff, dl, MVT::i32));
5299 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5300 DAG.getConstant(0x3f800000, dl, MVT::i32));
5301 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5302}
5303
5304/// GetExponent - Get the exponent:
5305///
5306/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5307///
5308/// where Op is the hexadecimal representation of floating point value.
5310 const TargetLowering &TLI, const SDLoc &dl) {
5311 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5312 DAG.getConstant(0x7f800000, dl, MVT::i32));
5313 SDValue t1 = DAG.getNode(
5314 ISD::SRL, dl, MVT::i32, t0,
5315 DAG.getConstant(23, dl,
5316 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5317 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5318 DAG.getConstant(127, dl, MVT::i32));
5319 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5320}
5321
5322/// getF32Constant - Get 32-bit floating point constant.
5324 const SDLoc &dl) {
5325 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5326 MVT::f32);
5327}
5328
5330 SelectionDAG &DAG) {
5331 // TODO: What fast-math-flags should be set on the floating-point nodes?
5332
5333 // IntegerPartOfX = ((int32_t)(t0);
5334 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5335
5336 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5337 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5338 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5339
5340 // IntegerPartOfX <<= 23;
5341 IntegerPartOfX =
5342 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5343 DAG.getConstant(23, dl,
5345 MVT::i32, DAG.getDataLayout())));
5346
5347 SDValue TwoToFractionalPartOfX;
5348 if (LimitFloatPrecision <= 6) {
5349 // For floating-point precision of 6:
5350 //
5351 // TwoToFractionalPartOfX =
5352 // 0.997535578f +
5353 // (0.735607626f + 0.252464424f * x) * x;
5354 //
5355 // error 0.0144103317, which is 6 bits
5356 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5357 getF32Constant(DAG, 0x3e814304, dl));
5358 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5359 getF32Constant(DAG, 0x3f3c50c8, dl));
5360 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5361 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5362 getF32Constant(DAG, 0x3f7f5e7e, dl));
5363 } else if (LimitFloatPrecision <= 12) {
5364 // For floating-point precision of 12:
5365 //
5366 // TwoToFractionalPartOfX =
5367 // 0.999892986f +
5368 // (0.696457318f +
5369 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5370 //
5371 // error 0.000107046256, which is 13 to 14 bits
5372 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5373 getF32Constant(DAG, 0x3da235e3, dl));
5374 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5375 getF32Constant(DAG, 0x3e65b8f3, dl));
5376 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5377 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5378 getF32Constant(DAG, 0x3f324b07, dl));
5379 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5380 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5381 getF32Constant(DAG, 0x3f7ff8fd, dl));
5382 } else { // LimitFloatPrecision <= 18
5383 // For floating-point precision of 18:
5384 //
5385 // TwoToFractionalPartOfX =
5386 // 0.999999982f +
5387 // (0.693148872f +
5388 // (0.240227044f +
5389 // (0.554906021e-1f +
5390 // (0.961591928e-2f +
5391 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5392 // error 2.47208000*10^(-7), which is better than 18 bits
5393 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5394 getF32Constant(DAG, 0x3924b03e, dl));
5395 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5396 getF32Constant(DAG, 0x3ab24b87, dl));
5397 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5398 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5399 getF32Constant(DAG, 0x3c1d8c17, dl));
5400 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5401 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5402 getF32Constant(DAG, 0x3d634a1d, dl));
5403 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5404 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5405 getF32Constant(DAG, 0x3e75fe14, dl));
5406 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5407 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5408 getF32Constant(DAG, 0x3f317234, dl));
5409 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5410 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5411 getF32Constant(DAG, 0x3f800000, dl));
5412 }
5413
5414 // Add the exponent into the result in integer domain.
5415 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5416 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5417 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5418}
5419
5420/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5421/// limited-precision mode.
5423 const TargetLowering &TLI, SDNodeFlags Flags) {
5424 if (Op.getValueType() == MVT::f32 &&
5426
5427 // Put the exponent in the right bit position for later addition to the
5428 // final result:
5429 //
5430 // t0 = Op * log2(e)
5431
5432 // TODO: What fast-math-flags should be set here?
5433 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5434 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5435 return getLimitedPrecisionExp2(t0, dl, DAG);
5436 }
5437
5438 // No special expansion.
5439 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5440}
5441
5442/// expandLog - Lower a log intrinsic. Handles the special sequences for
5443/// limited-precision mode.
5445 const TargetLowering &TLI, SDNodeFlags Flags) {
5446 // TODO: What fast-math-flags should be set on the floating-point nodes?
5447
5448 if (Op.getValueType() == MVT::f32 &&
5450 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5451
5452 // Scale the exponent by log(2).
5453 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5454 SDValue LogOfExponent =
5455 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5456 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5457
5458 // Get the significand and build it into a floating-point number with
5459 // exponent of 1.
5460 SDValue X = GetSignificand(DAG, Op1, dl);
5461
5462 SDValue LogOfMantissa;
5463 if (LimitFloatPrecision <= 6) {
5464 // For floating-point precision of 6:
5465 //
5466 // LogofMantissa =
5467 // -1.1609546f +
5468 // (1.4034025f - 0.23903021f * x) * x;
5469 //
5470 // error 0.0034276066, which is better than 8 bits
5471 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5472 getF32Constant(DAG, 0xbe74c456, dl));
5473 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5474 getF32Constant(DAG, 0x3fb3a2b1, dl));
5475 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5476 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5477 getF32Constant(DAG, 0x3f949a29, dl));
5478 } else if (LimitFloatPrecision <= 12) {
5479 // For floating-point precision of 12:
5480 //
5481 // LogOfMantissa =
5482 // -1.7417939f +
5483 // (2.8212026f +
5484 // (-1.4699568f +
5485 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5486 //
5487 // error 0.000061011436, which is 14 bits
5488 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5489 getF32Constant(DAG, 0xbd67b6d6, dl));
5490 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5491 getF32Constant(DAG, 0x3ee4f4b8, dl));
5492 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5493 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5494 getF32Constant(DAG, 0x3fbc278b, dl));
5495 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5496 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5497 getF32Constant(DAG, 0x40348e95, dl));
5498 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5499 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5500 getF32Constant(DAG, 0x3fdef31a, dl));
5501 } else { // LimitFloatPrecision <= 18
5502 // For floating-point precision of 18:
5503 //
5504 // LogOfMantissa =
5505 // -2.1072184f +
5506 // (4.2372794f +
5507 // (-3.7029485f +
5508 // (2.2781945f +
5509 // (-0.87823314f +
5510 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5511 //
5512 // error 0.0000023660568, which is better than 18 bits
5513 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5514 getF32Constant(DAG, 0xbc91e5ac, dl));
5515 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5516 getF32Constant(DAG, 0x3e4350aa, dl));
5517 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5518 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5519 getF32Constant(DAG, 0x3f60d3e3, dl));
5520 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5521 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5522 getF32Constant(DAG, 0x4011cdf0, dl));
5523 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5524 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5525 getF32Constant(DAG, 0x406cfd1c, dl));
5526 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5527 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5528 getF32Constant(DAG, 0x408797cb, dl));
5529 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5530 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5531 getF32Constant(DAG, 0x4006dcab, dl));
5532 }
5533
5534 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5535 }
5536
5537 // No special expansion.
5538 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5539}
5540
5541/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5542/// limited-precision mode.
5544 const TargetLowering &TLI, SDNodeFlags Flags) {
5545 // TODO: What fast-math-flags should be set on the floating-point nodes?
5546
5547 if (Op.getValueType() == MVT::f32 &&
5549 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5550
5551 // Get the exponent.
5552 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5553
5554 // Get the significand and build it into a floating-point number with
5555 // exponent of 1.
5556 SDValue X = GetSignificand(DAG, Op1, dl);
5557
5558 // Different possible minimax approximations of significand in
5559 // floating-point for various degrees of accuracy over [1,2].
5560 SDValue Log2ofMantissa;
5561 if (LimitFloatPrecision <= 6) {
5562 // For floating-point precision of 6:
5563 //
5564 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5565 //
5566 // error 0.0049451742, which is more than 7 bits
5567 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5568 getF32Constant(DAG, 0xbeb08fe0, dl));
5569 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5570 getF32Constant(DAG, 0x40019463, dl));
5571 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5572 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5573 getF32Constant(DAG, 0x3fd6633d, dl));
5574 } else if (LimitFloatPrecision <= 12) {
5575 // For floating-point precision of 12:
5576 //
5577 // Log2ofMantissa =
5578 // -2.51285454f +
5579 // (4.07009056f +
5580 // (-2.12067489f +
5581 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5582 //
5583 // error 0.0000876136000, which is better than 13 bits
5584 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5585 getF32Constant(DAG, 0xbda7262e, dl));
5586 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5587 getF32Constant(DAG, 0x3f25280b, dl));
5588 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5589 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5590 getF32Constant(DAG, 0x4007b923, dl));
5591 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5592 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5593 getF32Constant(DAG, 0x40823e2f, dl));
5594 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5595 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5596 getF32Constant(DAG, 0x4020d29c, dl));
5597 } else { // LimitFloatPrecision <= 18
5598 // For floating-point precision of 18:
5599 //
5600 // Log2ofMantissa =
5601 // -3.0400495f +
5602 // (6.1129976f +
5603 // (-5.3420409f +
5604 // (3.2865683f +
5605 // (-1.2669343f +
5606 // (0.27515199f -
5607 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5608 //
5609 // error 0.0000018516, which is better than 18 bits
5610 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5611 getF32Constant(DAG, 0xbcd2769e, dl));
5612 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5613 getF32Constant(DAG, 0x3e8ce0b9, dl));
5614 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5615 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5616 getF32Constant(DAG, 0x3fa22ae7, dl));
5617 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5618 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5619 getF32Constant(DAG, 0x40525723, dl));
5620 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5621 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5622 getF32Constant(DAG, 0x40aaf200, dl));
5623 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5624 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5625 getF32Constant(DAG, 0x40c39dad, dl));
5626 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5627 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5628 getF32Constant(DAG, 0x4042902c, dl));
5629 }
5630
5631 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5632 }
5633
5634 // No special expansion.
5635 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5636}
5637
5638/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5639/// limited-precision mode.
5641 const TargetLowering &TLI, SDNodeFlags Flags) {
5642 // TODO: What fast-math-flags should be set on the floating-point nodes?
5643
5644 if (Op.getValueType() == MVT::f32 &&
5646 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5647
5648 // Scale the exponent by log10(2) [0.30102999f].
5649 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5650 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5651 getF32Constant(DAG, 0x3e9a209a, dl));
5652
5653 // Get the significand and build it into a floating-point number with
5654 // exponent of 1.
5655 SDValue X = GetSignificand(DAG, Op1, dl);
5656
5657 SDValue Log10ofMantissa;
5658 if (LimitFloatPrecision <= 6) {
5659 // For floating-point precision of 6:
5660 //
5661 // Log10ofMantissa =
5662 // -0.50419619f +
5663 // (0.60948995f - 0.10380950f * x) * x;
5664 //
5665 // error 0.0014886165, which is 6 bits
5666 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5667 getF32Constant(DAG, 0xbdd49a13, dl));
5668 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5669 getF32Constant(DAG, 0x3f1c0789, dl));
5670 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5671 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5672 getF32Constant(DAG, 0x3f011300, dl));
5673 } else if (LimitFloatPrecision <= 12) {
5674 // For floating-point precision of 12:
5675 //
5676 // Log10ofMantissa =
5677 // -0.64831180f +
5678 // (0.91751397f +
5679 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5680 //
5681 // error 0.00019228036, which is better than 12 bits
5682 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5683 getF32Constant(DAG, 0x3d431f31, dl));
5684 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5685 getF32Constant(DAG, 0x3ea21fb2, dl));
5686 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5687 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5688 getF32Constant(DAG, 0x3f6ae232, dl));
5689 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5690 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5691 getF32Constant(DAG, 0x3f25f7c3, dl));
5692 } else { // LimitFloatPrecision <= 18
5693 // For floating-point precision of 18:
5694 //
5695 // Log10ofMantissa =
5696 // -0.84299375f +
5697 // (1.5327582f +
5698 // (-1.0688956f +
5699 // (0.49102474f +
5700 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5701 //
5702 // error 0.0000037995730, which is better than 18 bits
5703 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5704 getF32Constant(DAG, 0x3c5d51ce, dl));
5705 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5706 getF32Constant(DAG, 0x3e00685a, dl));
5707 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5708 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5709 getF32Constant(DAG, 0x3efb6798, dl));
5710 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5711 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5712 getF32Constant(DAG, 0x3f88d192, dl));
5713 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5714 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5715 getF32Constant(DAG, 0x3fc4316c, dl));
5716 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5717 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5718 getF32Constant(DAG, 0x3f57ce70, dl));
5719 }
5720
5721 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5722 }
5723
5724 // No special expansion.
5725 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5726}
5727
5728/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5729/// limited-precision mode.
5731 const TargetLowering &TLI, SDNodeFlags Flags) {
5732 if (Op.getValueType() == MVT::f32 &&
5734 return getLimitedPrecisionExp2(Op, dl, DAG);
5735
5736 // No special expansion.
5737 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5738}
5739
5740/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5741/// limited-precision mode with x == 10.0f.
5742static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5743 SelectionDAG &DAG, const TargetLowering &TLI,
5744 SDNodeFlags Flags) {
5745 bool IsExp10 = false;
5746 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5748 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5749 APFloat Ten(10.0f);
5750 IsExp10 = LHSC->isExactlyValue(Ten);
5751 }
5752 }
5753
5754 // TODO: What fast-math-flags should be set on the FMUL node?
5755 if (IsExp10) {
5756 // Put the exponent in the right bit position for later addition to the
5757 // final result:
5758 //
5759 // #define LOG2OF10 3.3219281f
5760 // t0 = Op * LOG2OF10;
5761 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5762 getF32Constant(DAG, 0x40549a78, dl));
5763 return getLimitedPrecisionExp2(t0, dl, DAG);
5764 }
5765
5766 // No special expansion.
5767 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5768}
5769
5770/// ExpandPowI - Expand a llvm.powi intrinsic.
5771static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5772 SelectionDAG &DAG) {
5773 // If RHS is a constant, we can expand this out to a multiplication tree if
5774 // it's beneficial on the target, otherwise we end up lowering to a call to
5775 // __powidf2 (for example).
5776 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5777 unsigned Val = RHSC->getSExtValue();
5778
5779 // powi(x, 0) -> 1.0
5780 if (Val == 0)
5781 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5782
5784 Val, DAG.shouldOptForSize())) {
5785 // Get the exponent as a positive value.
5786 if ((int)Val < 0)
5787 Val = -Val;
5788 // We use the simple binary decomposition method to generate the multiply
5789 // sequence. There are more optimal ways to do this (for example,
5790 // powi(x,15) generates one more multiply than it should), but this has
5791 // the benefit of being both really simple and much better than a libcall.
5792 SDValue Res; // Logically starts equal to 1.0
5793 SDValue CurSquare = LHS;
5794 // TODO: Intrinsics should have fast-math-flags that propagate to these
5795 // nodes.
5796 while (Val) {
5797 if (Val & 1) {
5798 if (Res.getNode())
5799 Res =
5800 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5801 else
5802 Res = CurSquare; // 1.0*CurSquare.
5803 }
5804
5805 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5806 CurSquare, CurSquare);
5807 Val >>= 1;
5808 }
5809
5810 // If the original was negative, invert the result, producing 1/(x*x*x).
5811 if (RHSC->getSExtValue() < 0)
5812 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5813 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5814 return Res;
5815 }
5816 }
5817
5818 // Otherwise, expand to a libcall.
5819 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5820}
5821
5822static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5823 SDValue LHS, SDValue RHS, SDValue Scale,
5824 SelectionDAG &DAG, const TargetLowering &TLI) {
5825 EVT VT = LHS.getValueType();
5826 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5827 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5828 LLVMContext &Ctx = *DAG.getContext();
5829
5830 // If the type is legal but the operation isn't, this node might survive all
5831 // the way to operation legalization. If we end up there and we do not have
5832 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5833 // node.
5834
5835 // Coax the legalizer into expanding the node during type legalization instead
5836 // by bumping the size by one bit. This will force it to Promote, enabling the
5837 // early expansion and avoiding the need to expand later.
5838
5839 // We don't have to do this if Scale is 0; that can always be expanded, unless
5840 // it's a saturating signed operation. Those can experience true integer
5841 // division overflow, a case which we must avoid.
5842
5843 // FIXME: We wouldn't have to do this (or any of the early
5844 // expansion/promotion) if it was possible to expand a libcall of an
5845 // illegal type during operation legalization. But it's not, so things
5846 // get a bit hacky.
5847 unsigned ScaleInt = Scale->getAsZExtVal();
5848 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5849 (TLI.isTypeLegal(VT) ||
5850 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5852 Opcode, VT, ScaleInt);
5853 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5854 EVT PromVT;
5855 if (VT.isScalarInteger())
5856 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5857 else if (VT.isVector()) {
5858 PromVT = VT.getVectorElementType();
5859 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5860 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5861 } else
5862 llvm_unreachable("Wrong VT for DIVFIX?");
5863 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5864 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5865 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5866 // For saturating operations, we need to shift up the LHS to get the
5867 // proper saturation width, and then shift down again afterwards.
5868 if (Saturating)
5869 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5870 DAG.getConstant(1, DL, ShiftTy));
5871 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5872 if (Saturating)
5873 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5874 DAG.getConstant(1, DL, ShiftTy));
5875 return DAG.getZExtOrTrunc(Res, DL, VT);
5876 }
5877 }
5878
5879 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5880}
5881
5882// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5883// bitcasted, or split argument. Returns a list of <Register, size in bits>
5884static void
5885getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5886 const SDValue &N) {
5887 switch (N.getOpcode()) {
5888 case ISD::CopyFromReg: {
5889 SDValue Op = N.getOperand(1);
5890 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5891 Op.getValueType().getSizeInBits());
5892 return;
5893 }
5894 case ISD::BITCAST:
5895 case ISD::AssertZext:
5896 case ISD::AssertSext:
5897 case ISD::TRUNCATE:
5898 getUnderlyingArgRegs(Regs, N.getOperand(0));
5899 return;
5900 case ISD::BUILD_PAIR:
5901 case ISD::BUILD_VECTOR:
5903 for (SDValue Op : N->op_values())
5904 getUnderlyingArgRegs(Regs, Op);
5905 return;
5906 default:
5907 return;
5908 }
5909}
5910
5911/// If the DbgValueInst is a dbg_value of a function argument, create the
5912/// corresponding DBG_VALUE machine instruction for it now. At the end of
5913/// instruction selection, they will be inserted to the entry BB.
5914/// We don't currently support this for variadic dbg_values, as they shouldn't
5915/// appear for function arguments or in the prologue.
5916bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5917 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5918 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5919 const Argument *Arg = dyn_cast<Argument>(V);
5920 if (!Arg)
5921 return false;
5922
5925
5926 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5927 // we've been asked to pursue.
5928 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5929 bool Indirect) {
5930 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5931 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5932 // pointing at the VReg, which will be patched up later.
5933 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5935 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
5936 /* isKill */ false, /* isDead */ false,
5937 /* isUndef */ false, /* isEarlyClobber */ false,
5938 /* SubReg */ 0, /* isDebug */ true)});
5939
5940 auto *NewDIExpr = FragExpr;
5941 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5942 // the DIExpression.
5943 if (Indirect)
5944 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5946 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
5947 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
5948 } else {
5949 // Create a completely standard DBG_VALUE.
5950 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5951 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5952 }
5953 };
5954
5955 if (Kind == FuncArgumentDbgValueKind::Value) {
5956 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5957 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5958 // the entry block.
5959 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5960 if (!IsInEntryBlock)
5961 return false;
5962
5963 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5964 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5965 // variable that also is a param.
5966 //
5967 // Although, if we are at the top of the entry block already, we can still
5968 // emit using ArgDbgValue. This might catch some situations when the
5969 // dbg.value refers to an argument that isn't used in the entry block, so
5970 // any CopyToReg node would be optimized out and the only way to express
5971 // this DBG_VALUE is by using the physical reg (or FI) as done in this
5972 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
5973 // we should only emit as ArgDbgValue if the Variable is an argument to the
5974 // current function, and the dbg.value intrinsic is found in the entry
5975 // block.
5976 bool VariableIsFunctionInputArg = Variable->isParameter() &&
5977 !DL->getInlinedAt();
5978 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5979 if (!IsInPrologue && !VariableIsFunctionInputArg)
5980 return false;
5981
5982 // Here we assume that a function argument on IR level only can be used to
5983 // describe one input parameter on source level. If we for example have
5984 // source code like this
5985 //
5986 // struct A { long x, y; };
5987 // void foo(struct A a, long b) {
5988 // ...
5989 // b = a.x;
5990 // ...
5991 // }
5992 //
5993 // and IR like this
5994 //
5995 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
5996 // entry:
5997 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5998 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5999 // call void @llvm.dbg.value(metadata i32 %b, "b",
6000 // ...
6001 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6002 // ...
6003 //
6004 // then the last dbg.value is describing a parameter "b" using a value that
6005 // is an argument. But since we already has used %a1 to describe a parameter
6006 // we should not handle that last dbg.value here (that would result in an
6007 // incorrect hoisting of the DBG_VALUE to the function entry).
6008 // Notice that we allow one dbg.value per IR level argument, to accommodate
6009 // for the situation with fragments above.
6010 // If there is no node for the value being handled, we return true to skip
6011 // the normal generation of debug info, as it would kill existing debug
6012 // info for the parameter in case of duplicates.
6013 if (VariableIsFunctionInputArg) {
6014 unsigned ArgNo = Arg->getArgNo();
6015 if (ArgNo >= FuncInfo.DescribedArgs.size())
6016 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6017 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6018 return !NodeMap[V].getNode();
6019 FuncInfo.DescribedArgs.set(ArgNo);
6020 }
6021 }
6022
6023 bool IsIndirect = false;
6024 std::optional<MachineOperand> Op;
6025 // Some arguments' frame index is recorded during argument lowering.
6026 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6027 if (FI != std::numeric_limits<int>::max())
6029
6031 if (!Op && N.getNode()) {
6032 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6033 Register Reg;
6034 if (ArgRegsAndSizes.size() == 1)
6035 Reg = ArgRegsAndSizes.front().first;
6036
6037 if (Reg && Reg.isVirtual()) {
6039 Register PR = RegInfo.getLiveInPhysReg(Reg);
6040 if (PR)
6041 Reg = PR;
6042 }
6043 if (Reg) {
6044 Op = MachineOperand::CreateReg(Reg, false);
6045 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6046 }
6047 }
6048
6049 if (!Op && N.getNode()) {
6050 // Check if frame index is available.
6051 SDValue LCandidate = peekThroughBitcasts(N);
6052 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6053 if (FrameIndexSDNode *FINode =
6054 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6055 Op = MachineOperand::CreateFI(FINode->getIndex());
6056 }
6057
6058 if (!Op) {
6059 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6060 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
6061 SplitRegs) {
6062 unsigned Offset = 0;
6063 for (const auto &RegAndSize : SplitRegs) {
6064 // If the expression is already a fragment, the current register
6065 // offset+size might extend beyond the fragment. In this case, only
6066 // the register bits that are inside the fragment are relevant.
6067 int RegFragmentSizeInBits = RegAndSize.second;
6068 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6069 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6070 // The register is entirely outside the expression fragment,
6071 // so is irrelevant for debug info.
6072 if (Offset >= ExprFragmentSizeInBits)
6073 break;
6074 // The register is partially outside the expression fragment, only
6075 // the low bits within the fragment are relevant for debug info.
6076 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6077 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6078 }
6079 }
6080
6081 auto FragmentExpr = DIExpression::createFragmentExpression(
6082 Expr, Offset, RegFragmentSizeInBits);
6083 Offset += RegAndSize.second;
6084 // If a valid fragment expression cannot be created, the variable's
6085 // correct value cannot be determined and so it is set as Undef.
6086 if (!FragmentExpr) {
6088 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
6089 DAG.AddDbgValue(SDV, false);
6090 continue;
6091 }
6092 MachineInstr *NewMI =
6093 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6094 Kind != FuncArgumentDbgValueKind::Value);
6095 FuncInfo.ArgDbgValues.push_back(NewMI);
6096 }
6097 };
6098
6099 // Check if ValueMap has reg number.
6101 VMI = FuncInfo.ValueMap.find(V);
6102 if (VMI != FuncInfo.ValueMap.end()) {
6103 const auto &TLI = DAG.getTargetLoweringInfo();
6104 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6105 V->getType(), std::nullopt);
6106 if (RFV.occupiesMultipleRegs()) {
6107 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6108 return true;
6109 }
6110
6111 Op = MachineOperand::CreateReg(VMI->second, false);
6112 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6113 } else if (ArgRegsAndSizes.size() > 1) {
6114 // This was split due to the calling convention, and no virtual register
6115 // mapping exists for the value.
6116 splitMultiRegDbgValue(ArgRegsAndSizes);
6117 return true;
6118 }
6119 }
6120
6121 if (!Op)
6122 return false;
6123
6125 "Expected inlined-at fields to agree");
6126 MachineInstr *NewMI = nullptr;
6127
6128 if (Op->isReg())
6129 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6130 else
6131 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6132 Variable, Expr);
6133
6134 // Otherwise, use ArgDbgValues.
6135 FuncInfo.ArgDbgValues.push_back(NewMI);
6136 return true;
6137}
6138
6139/// Return the appropriate SDDbgValue based on N.
6140SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6141 DILocalVariable *Variable,
6142 DIExpression *Expr,
6143 const DebugLoc &dl,
6144 unsigned DbgSDNodeOrder) {
6145 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6146 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6147 // stack slot locations.
6148 //
6149 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6150 // debug values here after optimization:
6151 //
6152 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6153 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6154 //
6155 // Both describe the direct values of their associated variables.
6156 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6157 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6158 }
6159 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6160 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6161}
6162
6163static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6164 switch (Intrinsic) {
6165 case Intrinsic::smul_fix:
6166 return ISD::SMULFIX;
6167 case Intrinsic::umul_fix:
6168 return ISD::UMULFIX;
6169 case Intrinsic::smul_fix_sat:
6170 return ISD::SMULFIXSAT;
6171 case Intrinsic::umul_fix_sat:
6172 return ISD::UMULFIXSAT;
6173 case Intrinsic::sdiv_fix:
6174 return ISD::SDIVFIX;
6175 case Intrinsic::udiv_fix:
6176 return ISD::UDIVFIX;
6177 case Intrinsic::sdiv_fix_sat:
6178 return ISD::SDIVFIXSAT;
6179 case Intrinsic::udiv_fix_sat:
6180 return ISD::UDIVFIXSAT;
6181 default:
6182 llvm_unreachable("Unhandled fixed point intrinsic");
6183 }
6184}
6185
6186void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
6187 const char *FunctionName) {
6188 assert(FunctionName && "FunctionName must not be nullptr");
6190 FunctionName,
6192 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
6193}
6194
6195/// Given a @llvm.call.preallocated.setup, return the corresponding
6196/// preallocated call.
6197static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6198 assert(cast<CallBase>(PreallocatedSetup)
6200 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6201 "expected call_preallocated_setup Value");
6202 for (const auto *U : PreallocatedSetup->users()) {
6203 auto *UseCall = cast<CallBase>(U);
6204 const Function *Fn = UseCall->getCalledFunction();
6205 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6206 return UseCall;
6207 }
6208 }
6209 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6210}
6211
6212/// If DI is a debug value with an EntryValue expression, lower it using the
6213/// corresponding physical register of the associated Argument value
6214/// (guaranteed to exist by the verifier).
6215bool SelectionDAGBuilder::visitEntryValueDbgValue(
6216 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6217 DIExpression *Expr, DebugLoc DbgLoc) {
6218 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6219 return false;
6220
6221 // These properties are guaranteed by the verifier.
6222 const Argument *Arg = cast<Argument>(Values[0]);
6223 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6224
6225 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6226 if (ArgIt == FuncInfo.ValueMap.end()) {
6227 LLVM_DEBUG(
6228 dbgs() << "Dropping dbg.value: expression is entry_value but "
6229 "couldn't find an associated register for the Argument\n");
6230 return true;
6231 }
6232 Register ArgVReg = ArgIt->getSecond();
6233
6234 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6235 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6237 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6238 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6239 return true;
6240 }
6241 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6242 "couldn't find a physical register\n");
6243 return true;
6244}
6245
6246/// Lower the call to the specified intrinsic function.
6247void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6248 unsigned Intrinsic) {
6249 SDLoc sdl = getCurSDLoc();
6250 switch (Intrinsic) {
6251 case Intrinsic::experimental_convergence_anchor:
6252 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6253 break;
6254 case Intrinsic::experimental_convergence_entry:
6255 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6256 break;
6257 case Intrinsic::experimental_convergence_loop: {
6258 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6259 auto *Token = Bundle->Inputs[0].get();
6260 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6261 getValue(Token)));
6262 break;
6263 }
6264 }
6265}
6266
6267/// Lower the call to the specified intrinsic function.
6268void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6269 unsigned Intrinsic) {
6271 SDLoc sdl = getCurSDLoc();
6272 DebugLoc dl = getCurDebugLoc();
6273 SDValue Res;
6274
6276 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6277 Flags.copyFMF(*FPOp);
6278
6279 switch (Intrinsic) {
6280 default:
6281 // By default, turn this into a target intrinsic node.
6282 visitTargetIntrinsic(I, Intrinsic);
6283 return;
6284 case Intrinsic::vscale: {
6285 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6286 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6287 return;
6288 }
6289 case Intrinsic::vastart: visitVAStart(I); return;
6290 case Intrinsic::vaend: visitVAEnd(I); return;
6291 case Intrinsic::vacopy: visitVACopy(I); return;
6292 case Intrinsic::returnaddress:
6294 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6295 getValue(I.getArgOperand(0))));
6296 return;
6297 case Intrinsic::addressofreturnaddress:
6298 setValue(&I,
6300 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6301 return;
6302 case Intrinsic::sponentry:
6303 setValue(&I,
6305 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6306 return;
6307 case Intrinsic::frameaddress:
6310 getValue(I.getArgOperand(0))));
6311 return;
6312 case Intrinsic::read_volatile_register:
6313 case Intrinsic::read_register: {
6314 Value *Reg = I.getArgOperand(0);
6315 SDValue Chain = getRoot();
6317 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6318 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6319 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6320 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6321 setValue(&I, Res);
6322 DAG.setRoot(Res.getValue(1));
6323 return;
6324 }
6325 case Intrinsic::write_register: {
6326 Value *Reg = I.getArgOperand(0);
6327 Value *RegValue = I.getArgOperand(1);
6328 SDValue Chain = getRoot();
6330 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6331 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6332 RegName, getValue(RegValue)));
6333 return;
6334 }
6335 case Intrinsic::memcpy: {
6336 const auto &MCI = cast<MemCpyInst>(I);
6337 SDValue Op1 = getValue(I.getArgOperand(0));
6338 SDValue Op2 = getValue(I.getArgOperand(1));
6339 SDValue Op3 = getValue(I.getArgOperand(2));
6340 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
6341 Align DstAlign = MCI.getDestAlign().valueOrOne();
6342 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6343 Align Alignment = std::min(DstAlign, SrcAlign);
6344 bool isVol = MCI.isVolatile();
6345 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6346 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6347 // node.
6348 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6349 SDValue MC = DAG.getMemcpy(
6350 Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6351 /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
6352 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6353 updateDAGForMaybeTailCall(MC);
6354 return;
6355 }
6356 case Intrinsic::memcpy_inline: {
6357 const auto &MCI = cast<MemCpyInlineInst>(I);
6358 SDValue Dst = getValue(I.getArgOperand(0));
6359 SDValue Src = getValue(I.getArgOperand(1));
6360 SDValue Size = getValue(I.getArgOperand(2));
6361 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
6362 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6363 Align DstAlign = MCI.getDestAlign().valueOrOne();
6364 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6365 Align Alignment = std::min(DstAlign, SrcAlign);
6366 bool isVol = MCI.isVolatile();
6367 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6368 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6369 // node.
6370 SDValue MC = DAG.getMemcpy(
6371 getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
6372 /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
6373 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6374 updateDAGForMaybeTailCall(MC);
6375 return;
6376 }
6377 case Intrinsic::memset: {
6378 const auto &MSI = cast<MemSetInst>(I);
6379 SDValue Op1 = getValue(I.getArgOperand(0));
6380 SDValue Op2 = getValue(I.getArgOperand(1));
6381 SDValue Op3 = getValue(I.getArgOperand(2));
6382 // @llvm.memset defines 0 and 1 to both mean no alignment.
6383 Align Alignment = MSI.getDestAlign().valueOrOne();
6384 bool isVol = MSI.isVolatile();
6385 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6386 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6387 SDValue MS = DAG.getMemset(
6388 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6389 isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6390 updateDAGForMaybeTailCall(MS);
6391 return;
6392 }
6393 case Intrinsic::memset_inline: {
6394 const auto &MSII = cast<MemSetInlineInst>(I);
6395 SDValue Dst = getValue(I.getArgOperand(0));
6396 SDValue Value = getValue(I.getArgOperand(1));
6397 SDValue Size = getValue(I.getArgOperand(2));
6398 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6399 // @llvm.memset defines 0 and 1 to both mean no alignment.
6400 Align DstAlign = MSII.getDestAlign().valueOrOne();
6401 bool isVol = MSII.isVolatile();
6402 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6403 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6404 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6405 /* AlwaysInline */ true, isTC,
6406 MachinePointerInfo(I.getArgOperand(0)),
6407 I.getAAMetadata());
6408 updateDAGForMaybeTailCall(MC);
6409 return;
6410 }
6411 case Intrinsic::memmove: {
6412 const auto &MMI = cast<MemMoveInst>(I);
6413 SDValue Op1 = getValue(I.getArgOperand(0));
6414 SDValue Op2 = getValue(I.getArgOperand(1));
6415 SDValue Op3 = getValue(I.getArgOperand(2));
6416 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6417 Align DstAlign = MMI.getDestAlign().valueOrOne();
6418 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6419 Align Alignment = std::min(DstAlign, SrcAlign);
6420 bool isVol = MMI.isVolatile();
6421 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6422 // FIXME: Support passing different dest/src alignments to the memmove DAG
6423 // node.
6424 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6425 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6426 isTC, MachinePointerInfo(I.getArgOperand(0)),
6427 MachinePointerInfo(I.getArgOperand(1)),
6428 I.getAAMetadata(), AA);
6429 updateDAGForMaybeTailCall(MM);
6430 return;
6431 }
6432 case Intrinsic::memcpy_element_unordered_atomic: {
6433 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6434 SDValue Dst = getValue(MI.getRawDest());
6435 SDValue Src = getValue(MI.getRawSource());
6436 SDValue Length = getValue(MI.getLength());
6437
6438 Type *LengthTy = MI.getLength()->getType();
6439 unsigned ElemSz = MI.getElementSizeInBytes();
6440 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6441 SDValue MC =
6442 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6443 isTC, MachinePointerInfo(MI.getRawDest()),
6444 MachinePointerInfo(MI.getRawSource()));
6445 updateDAGForMaybeTailCall(MC);
6446 return;
6447 }
6448 case Intrinsic::memmove_element_unordered_atomic: {
6449 auto &MI = cast<AtomicMemMoveInst>(I);
6450 SDValue Dst = getValue(MI.getRawDest());
6451 SDValue Src = getValue(MI.getRawSource());
6452 SDValue Length = getValue(MI.getLength());
6453
6454 Type *LengthTy = MI.getLength()->getType();
6455 unsigned ElemSz = MI.getElementSizeInBytes();
6456 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6457 SDValue MC =
6458 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6459 isTC, MachinePointerInfo(MI.getRawDest()),
6460 MachinePointerInfo(MI.getRawSource()));
6461 updateDAGForMaybeTailCall(MC);
6462 return;
6463 }
6464 case Intrinsic::memset_element_unordered_atomic: {
6465 auto &MI = cast<AtomicMemSetInst>(I);
6466 SDValue Dst = getValue(MI.getRawDest());
6467 SDValue Val = getValue(MI.getValue());
6468 SDValue Length = getValue(MI.getLength());
6469
6470 Type *LengthTy = MI.getLength()->getType();
6471 unsigned ElemSz = MI.getElementSizeInBytes();
6472 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6473 SDValue MC =
6474 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6475 isTC, MachinePointerInfo(MI.getRawDest()));
6476 updateDAGForMaybeTailCall(MC);
6477 return;
6478 }
6479 case Intrinsic::call_preallocated_setup: {
6480 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6481 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6482 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6483 getRoot(), SrcValue);
6484 setValue(&I, Res);
6485 DAG.setRoot(Res);
6486 return;
6487 }
6488 case Intrinsic::call_preallocated_arg: {
6489 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6490 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6491 SDValue Ops[3];
6492 Ops[0] = getRoot();
6493 Ops[1] = SrcValue;
6494 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6495 MVT::i32); // arg index
6496 SDValue Res = DAG.getNode(
6498 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6499 setValue(&I, Res);
6500 DAG.setRoot(Res.getValue(1));
6501 return;
6502 }
6503 case Intrinsic::dbg_declare: {
6504 const auto &DI = cast<DbgDeclareInst>(I);
6505 // Debug intrinsics are handled separately in assignment tracking mode.
6506 // Some intrinsics are handled right after Argument lowering.
6507 if (AssignmentTrackingEnabled ||
6509 return;
6510 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6511 DILocalVariable *Variable = DI.getVariable();
6512 DIExpression *Expression = DI.getExpression();
6514 // Assume dbg.declare can not currently use DIArgList, i.e.
6515 // it is non-variadic.
6516 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6517 handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
6518 DI.getDebugLoc());
6519 return;
6520 }
6521 case Intrinsic::dbg_label: {
6522 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6523 DILabel *Label = DI.getLabel();
6524 assert(Label && "Missing label");
6525
6526 SDDbgLabel *SDV;
6527 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6528 DAG.AddDbgLabel(SDV);
6529 return;
6530 }
6531 case Intrinsic::dbg_assign: {
6532 // Debug intrinsics are handled seperately in assignment tracking mode.
6533 if (AssignmentTrackingEnabled)
6534 return;
6535 // If assignment tracking hasn't been enabled then fall through and treat
6536 // the dbg.assign as a dbg.value.
6537 [[fallthrough]];
6538 }
6539 case Intrinsic::dbg_value: {
6540 // Debug intrinsics are handled seperately in assignment tracking mode.
6541 if (AssignmentTrackingEnabled)
6542 return;
6543 const DbgValueInst &DI = cast<DbgValueInst>(I);
6544 assert(DI.getVariable() && "Missing variable");
6545
6546 DILocalVariable *Variable = DI.getVariable();
6549
6550 if (DI.isKillLocation()) {
6551 handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
6552 return;
6553 }
6554
6556 if (Values.empty())
6557 return;
6558
6559 bool IsVariadic = DI.hasArgList();
6560 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6561 SDNodeOrder, IsVariadic))
6562 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
6563 DI.getDebugLoc(), SDNodeOrder);
6564 return;
6565 }
6566
6567 case Intrinsic::eh_typeid_for: {
6568 // Find the type id for the given typeinfo.
6569 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6570 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6571 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6572 setValue(&I, Res);
6573 return;
6574 }
6575
6576 case Intrinsic::eh_return_i32:
6577 case Intrinsic::eh_return_i64:
6580 MVT::Other,
6582 getValue(I.getArgOperand(0)),
6583 getValue(I.getArgOperand(1))));
6584 return;
6585 case Intrinsic::eh_unwind_init:
6587 return;
6588 case Intrinsic::eh_dwarf_cfa:
6591 getValue(I.getArgOperand(0))));
6592 return;
6593 case Intrinsic::eh_sjlj_callsite: {
6595 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6596 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6597
6599 return;
6600 }
6601 case Intrinsic::eh_sjlj_functioncontext: {
6602 // Get and store the index of the function context.
6604 AllocaInst *FnCtx =
6605 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6606 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6608 return;
6609 }
6610 case Intrinsic::eh_sjlj_setjmp: {
6611 SDValue Ops[2];
6612 Ops[0] = getRoot();
6613 Ops[1] = getValue(I.getArgOperand(0));
6615 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6616 setValue(&I, Op.getValue(0));
6617 DAG.setRoot(Op.getValue(1));
6618 return;
6619 }
6620 case Intrinsic::eh_sjlj_longjmp:
6621 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6622 getRoot(), getValue(I.getArgOperand(0))));
6623 return;
6624 case Intrinsic::eh_sjlj_setup_dispatch:
6626 getRoot()));
6627 return;
6628 case Intrinsic::masked_gather:
6629 visitMaskedGather(I);
6630 return;
6631 case Intrinsic::masked_load:
6632 visitMaskedLoad(I);
6633 return;
6634 case Intrinsic::masked_scatter:
6635 visitMaskedScatter(I);
6636 return;
6637 case Intrinsic::masked_store:
6638 visitMaskedStore(I);
6639 return;
6640 case Intrinsic::masked_expandload:
6641 visitMaskedLoad(I, true /* IsExpanding */);
6642 return;
6643 case Intrinsic::masked_compressstore:
6644 visitMaskedStore(I, true /* IsCompressing */);
6645 return;
6646 case Intrinsic::powi:
6647 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6648 getValue(I.getArgOperand(1)), DAG));
6649 return;
6650 case Intrinsic::log:
6651 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6652 return;
6653 case Intrinsic::log2:
6654 setValue(&I,
6655 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6656 return;
6657 case Intrinsic::log10:
6658 setValue(&I,
6659 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6660 return;
6661 case Intrinsic::exp:
6662 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6663 return;
6664 case Intrinsic::exp2:
6665 setValue(&I,
6666 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6667 return;
6668 case Intrinsic::pow:
6669 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6670 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6671 return;
6672 case Intrinsic::sqrt:
6673 case Intrinsic::fabs:
6674 case Intrinsic::sin:
6675 case Intrinsic::cos:
6676 case Intrinsic::exp10:
6677 case Intrinsic::floor:
6678 case Intrinsic::ceil:
6679 case Intrinsic::trunc:
6680 case Intrinsic::rint:
6681 case Intrinsic::nearbyint:
6682 case Intrinsic::round:
6683 case Intrinsic::roundeven:
6684 case Intrinsic::canonicalize: {
6685 unsigned Opcode;
6686 switch (Intrinsic) {
6687 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6688 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6689 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6690 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6691 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6692 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6693 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6694 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6695 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6696 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6697 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6698 case Intrinsic::round: Opcode = ISD::FROUND; break;
6699 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6700 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6701 }
6702
6703 setValue(&I, DAG.getNode(Opcode, sdl,
6704 getValue(I.getArgOperand(0)).getValueType(),
6705 getValue(I.getArgOperand(0)), Flags));
6706 return;
6707 }
6708 case Intrinsic::lround:
6709 case Intrinsic::llround:
6710 case Intrinsic::lrint:
6711 case Intrinsic::llrint: {
6712 unsigned Opcode;
6713 switch (Intrinsic) {
6714 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6715 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6716 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6717 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6718 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6719 }
6720
6721 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6722 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6723 getValue(I.getArgOperand(0))));
6724 return;
6725 }
6726 case Intrinsic::minnum:
6728 getValue(I.getArgOperand(0)).getValueType(),
6729 getValue(I.getArgOperand(0)),
6730 getValue(I.getArgOperand(1)), Flags));
6731 return;
6732 case Intrinsic::maxnum:
6734 getValue(I.getArgOperand(0)).getValueType(),
6735 getValue(I.getArgOperand(0)),
6736 getValue(I.getArgOperand(1)), Flags));
6737 return;
6738 case Intrinsic::minimum:
6740 getValue(I.getArgOperand(0)).getValueType(),
6741 getValue(I.getArgOperand(0)),
6742 getValue(I.getArgOperand(1)), Flags));
6743 return;
6744 case Intrinsic::maximum:
6746 getValue(I.getArgOperand(0)).getValueType(),
6747 getValue(I.getArgOperand(0)),
6748 getValue(I.getArgOperand(1)), Flags));
6749 return;
6750 case Intrinsic::copysign:
6752 getValue(I.getArgOperand(0)).getValueType(),
6753 getValue(I.getArgOperand(0)),
6754 getValue(I.getArgOperand(1)), Flags));
6755 return;
6756 case Intrinsic::ldexp:
6758 getValue(I.getArgOperand(0)).getValueType(),
6759 getValue(I.getArgOperand(0)),
6760 getValue(I.getArgOperand(1)), Flags));
6761 return;
6762 case Intrinsic::frexp: {
6763 SmallVector<EVT, 2> ValueVTs;
6764 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6765 SDVTList VTs = DAG.getVTList(ValueVTs);
6766 setValue(&I,
6767 DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0))));
6768 return;
6769 }
6770 case Intrinsic::arithmetic_fence: {
6772 getValue(I.getArgOperand(0)).getValueType(),
6773 getValue(I.getArgOperand(0)), Flags));
6774 return;
6775 }
6776 case Intrinsic::fma:
6778 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6779 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6780 getValue(I.getArgOperand(2)), Flags));
6781 return;
6782#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6783 case Intrinsic::INTRINSIC:
6784#include "llvm/IR/ConstrainedOps.def"
6785 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6786 return;
6787#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6788#include "llvm/IR/VPIntrinsics.def"
6789 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6790 return;
6791 case Intrinsic::fptrunc_round: {
6792 // Get the last argument, the metadata and convert it to an integer in the
6793 // call
6794 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6795 std::optional<RoundingMode> RoundMode =
6796 convertStrToRoundingMode(cast<MDString>(MD)->getString());
6797
6798 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6799
6800 // Propagate fast-math-flags from IR to node(s).
6802 Flags.copyFMF(*cast<FPMathOperator>(&I));
6803 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6804
6806 Result = DAG.getNode(
6807 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6808 DAG.getTargetConstant((int)*RoundMode, sdl,
6810 setValue(&I, Result);
6811
6812 return;
6813 }
6814 case Intrinsic::fmuladd: {
6815 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6818 setValue(&I, DAG.getNode(ISD::FMA, sdl,
6819 getValue(I.getArgOperand(0)).getValueType(),
6820 getValue(I.getArgOperand(0)),
6821 getValue(I.getArgOperand(1)),
6822 getValue(I.getArgOperand(2)), Flags));
6823 } else {
6824 // TODO: Intrinsic calls should have fast-math-flags.
6826 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6827 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6829 getValue(I.getArgOperand(0)).getValueType(),
6830 Mul, getValue(I.getArgOperand(2)), Flags);
6831 setValue(&I, Add);
6832 }
6833 return;
6834 }
6835 case Intrinsic::convert_to_fp16:
6836 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6837 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6838 getValue(I.getArgOperand(0)),
6839 DAG.getTargetConstant(0, sdl,
6840 MVT::i32))));
6841 return;
6842 case Intrinsic::convert_from_fp16:
6844 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6845 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6846 getValue(I.getArgOperand(0)))));
6847 return;
6848 case Intrinsic::fptosi_sat: {
6849 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6851 getValue(I.getArgOperand(0)),
6853 return;
6854 }
6855 case Intrinsic::fptoui_sat: {
6856 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6858 getValue(I.getArgOperand(0)),
6860 return;
6861 }
6862 case Intrinsic::set_rounding:
6863 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6864 {getRoot(), getValue(I.getArgOperand(0))});
6865 setValue(&I, Res);
6866 DAG.setRoot(Res.getValue(0));
6867 return;
6868 case Intrinsic::is_fpclass: {
6869 const DataLayout DLayout = DAG.getDataLayout();
6870 EVT DestVT = TLI.getValueType(DLayout, I.getType());
6871 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6872 FPClassTest Test = static_cast<FPClassTest>(
6873 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
6875 const Function &F = MF.getFunction();
6876 SDValue Op = getValue(I.getArgOperand(0));
6878 Flags.setNoFPExcept(
6879 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6880 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6881 // expansion can use illegal types. Making expansion early allows
6882 // legalizing these types prior to selection.
6883 if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6884 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6885 setValue(&I, Result);
6886 return;
6887 }
6888
6889 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6890 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6891 setValue(&I, V);
6892 return;
6893 }
6894 case Intrinsic::get_fpenv: {
6895 const DataLayout DLayout = DAG.getDataLayout();
6896 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
6897 Align TempAlign = DAG.getEVTAlign(EnvVT);
6898 SDValue Chain = getRoot();
6899 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
6900 // and temporary storage in stack.
6901 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
6902 Res = DAG.getNode(
6903 ISD::GET_FPENV, sdl,
6904 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6905 MVT::Other),
6906 Chain);
6907 } else {
6908 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6909 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6910 auto MPI =
6914 TempAlign);
6915 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6916 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
6917 }
6918 setValue(&I, Res);
6919 DAG.setRoot(Res.getValue(1));
6920 return;
6921 }
6922 case Intrinsic::set_fpenv: {
6923 const DataLayout DLayout = DAG.getDataLayout();
6924 SDValue Env = getValue(I.getArgOperand(0));
6925 EVT EnvVT = Env.getValueType();
6926 Align TempAlign = DAG.getEVTAlign(EnvVT);
6927 SDValue Chain = getRoot();
6928 // If SET_FPENV is custom or legal, use it. Otherwise use loading
6929 // environment from memory.
6930 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
6931 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
6932 } else {
6933 // Allocate space in stack, copy environment bits into it and use this
6934 // memory in SET_FPENV_MEM.
6935 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6936 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6937 auto MPI =
6939 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
6943 TempAlign);
6944 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6945 }
6946 DAG.setRoot(Chain);
6947 return;
6948 }
6949 case Intrinsic::reset_fpenv:
6950 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
6951 return;
6952 case Intrinsic::get_fpmode:
6953 Res = DAG.getNode(
6954 ISD::GET_FPMODE, sdl,
6955 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6956 MVT::Other),
6957 DAG.getRoot());
6958 setValue(&I, Res);
6959 DAG.setRoot(Res.getValue(1));
6960 return;
6961 case Intrinsic::set_fpmode:
6962 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
6963 getValue(I.getArgOperand(0)));
6964 DAG.setRoot(Res);
6965 return;
6966 case Intrinsic::reset_fpmode: {
6967 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
6968 DAG.setRoot(Res);
6969 return;
6970 }
6971 case Intrinsic::pcmarker: {
6972 SDValue Tmp = getValue(I.getArgOperand(0));
6973 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6974 return;
6975 }
6976 case Intrinsic::readcyclecounter: {
6977 SDValue Op = getRoot();
6979 DAG.getVTList(MVT::i64, MVT::Other), Op);
6980 setValue(&I, Res);
6981 DAG.setRoot(Res.getValue(1));
6982 return;
6983 }
6984 case Intrinsic::readsteadycounter: {
6985 SDValue Op = getRoot();
6987 DAG.getVTList(MVT::i64, MVT::Other), Op);
6988 setValue(&I, Res);
6989 DAG.setRoot(Res.getValue(1));
6990 return;
6991 }
6992 case Intrinsic::bitreverse:
6994 getValue(I.getArgOperand(0)).getValueType(),
6995 getValue(I.getArgOperand(0))));
6996 return;
6997 case Intrinsic::bswap:
6999 getValue(I.getArgOperand(0)).getValueType(),
7000 getValue(I.getArgOperand(0))));
7001 return;
7002 case Intrinsic::cttz: {
7003 SDValue Arg = getValue(I.getArgOperand(0));
7004 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7005 EVT Ty = Arg.getValueType();
7007 sdl, Ty, Arg));
7008 return;
7009 }
7010 case Intrinsic::ctlz: {
7011 SDValue Arg = getValue(I.getArgOperand(0));
7012 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7013 EVT Ty = Arg.getValueType();
7015 sdl, Ty, Arg));
7016 return;
7017 }
7018 case Intrinsic::ctpop: {
7019 SDValue Arg = getValue(I.getArgOperand(0));
7020 EVT Ty = Arg.getValueType();
7021 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7022 return;
7023 }
7024 case Intrinsic::fshl:
7025 case Intrinsic::fshr: {
7026 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7027 SDValue X = getValue(I.getArgOperand(0));
7028 SDValue Y = getValue(I.getArgOperand(1));
7029 SDValue Z = getValue(I.getArgOperand(2));
7030 EVT VT = X.getValueType();
7031
7032 if (X == Y) {
7033 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7034 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7035 } else {
7036 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7037 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7038 }
7039 return;
7040 }
7041 case Intrinsic::sadd_sat: {
7042 SDValue Op1 = getValue(I.getArgOperand(0));
7043 SDValue Op2 = getValue(I.getArgOperand(1));
7044 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7045 return;
7046 }
7047 case Intrinsic::uadd_sat: {
7048 SDValue Op1 = getValue(I.getArgOperand(0));
7049 SDValue Op2 = getValue(I.getArgOperand(1));
7050 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7051 return;
7052 }
7053 case Intrinsic::ssub_sat: {
7054 SDValue Op1 = getValue(I.getArgOperand(0));
7055 SDValue Op2 = getValue(I.getArgOperand(1));
7056 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7057 return;
7058 }
7059 case Intrinsic::usub_sat: {
7060 SDValue Op1 = getValue(I.getArgOperand(0));
7061 SDValue Op2 = getValue(I.getArgOperand(1));
7062 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7063 return;
7064 }
7065 case Intrinsic::sshl_sat: {
7066 SDValue Op1 = getValue(I.getArgOperand(0));
7067 SDValue Op2 = getValue(I.getArgOperand(1));
7068 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7069 return;
7070 }
7071 case Intrinsic::ushl_sat: {
7072 SDValue Op1 = getValue(I.getArgOperand(0));
7073 SDValue Op2 = getValue(I.getArgOperand(1));
7074 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7075 return;
7076 }
7077 case Intrinsic::smul_fix:
7078 case Intrinsic::umul_fix:
7079 case Intrinsic::smul_fix_sat:
7080 case Intrinsic::umul_fix_sat: {
7081 SDValue Op1 = getValue(I.getArgOperand(0));
7082 SDValue Op2 = getValue(I.getArgOperand(1));
7083 SDValue Op3 = getValue(I.getArgOperand(2));
7085 Op1.getValueType(), Op1, Op2, Op3));
7086 return;
7087 }
7088 case Intrinsic::sdiv_fix:
7089 case Intrinsic::udiv_fix:
7090 case Intrinsic::sdiv_fix_sat:
7091 case Intrinsic::udiv_fix_sat: {
7092 SDValue Op1 = getValue(I.getArgOperand(0));
7093 SDValue Op2 = getValue(I.getArgOperand(1));
7094 SDValue Op3 = getValue(I.getArgOperand(2));
7096 Op1, Op2, Op3, DAG, TLI));
7097 return;
7098 }
7099 case Intrinsic::smax: {
7100 SDValue Op1 = getValue(I.getArgOperand(0));
7101 SDValue Op2 = getValue(I.getArgOperand(1));
7102 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7103 return;
7104 }
7105 case Intrinsic::smin: {
7106 SDValue Op1 = getValue(I.getArgOperand(0));
7107 SDValue Op2 = getValue(I.getArgOperand(1));
7108 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7109 return;
7110 }
7111 case Intrinsic::umax: {
7112 SDValue Op1 = getValue(I.getArgOperand(0));
7113 SDValue Op2 = getValue(I.getArgOperand(1));
7114 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7115 return;
7116 }
7117 case Intrinsic::umin: {
7118 SDValue Op1 = getValue(I.getArgOperand(0));
7119 SDValue Op2 = getValue(I.getArgOperand(1));
7120 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7121 return;
7122 }
7123 case Intrinsic::abs: {
7124 // TODO: Preserve "int min is poison" arg in SDAG?
7125 SDValue Op1 = getValue(I.getArgOperand(0));
7126 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7127 return;
7128 }
7129 case Intrinsic::stacksave: {
7130 SDValue Op = getRoot();
7131 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7132 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7133 setValue(&I, Res);
7134 DAG.setRoot(Res.getValue(1));
7135 return;
7136 }
7137 case Intrinsic::stackrestore:
7138 Res = getValue(I.getArgOperand(0));
7139 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7140 return;
7141 case Intrinsic::get_dynamic_area_offset: {
7142 SDValue Op = getRoot();
7143 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7144 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7145 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7146 // target.
7147 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7148 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
7149 " intrinsic!");
7151 Op);
7152 DAG.setRoot(Op);
7153 setValue(&I, Res);
7154 return;
7155 }
7156 case Intrinsic::stackguard: {
7158 const Module &M = *MF.getFunction().getParent();
7159 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7160 SDValue Chain = getRoot();
7161 if (TLI.useLoadStackGuardNode()) {
7162 Res = getLoadStackGuard(DAG, sdl, Chain);
7163 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7164 } else {
7165 const Value *Global = TLI.getSDagStackGuard(M);
7167 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7170 }
7171 if (TLI.useStackGuardXorFP())
7172 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7173 DAG.setRoot(Chain);
7174 setValue(&I, Res);
7175 return;
7176 }
7177 case Intrinsic::stackprotector: {
7178 // Emit code into the DAG to store the stack guard onto the stack.
7180 MachineFrameInfo &MFI = MF.getFrameInfo();
7181 SDValue Src, Chain = getRoot();
7182
7183 if (TLI.useLoadStackGuardNode())
7184 Src = getLoadStackGuard(DAG, sdl, Chain);
7185 else
7186 Src = getValue(I.getArgOperand(0)); // The guard's value.
7187
7188 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7189
7190 int FI = FuncInfo.StaticAllocaMap[Slot];
7191 MFI.setStackProtectorIndex(FI);
7192 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7193
7194 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7195
7196 // Store the stack protector onto the stack.
7197 Res = DAG.getStore(
7198 Chain, sdl, Src, FIN,
7201 setValue(&I, Res);
7202 DAG.setRoot(Res);
7203 return;
7204 }
7205 case Intrinsic::objectsize:
7206 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7207
7208 case Intrinsic::is_constant:
7209 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7210
7211 case Intrinsic::annotation:
7212 case Intrinsic::ptr_annotation:
7213 case Intrinsic::launder_invariant_group:
7214 case Intrinsic::strip_invariant_group:
7215 // Drop the intrinsic, but forward the value
7216 setValue(&I, getValue(I.getOperand(0)));
7217 return;
7218
7219 case Intrinsic::assume:
7220 case Intrinsic::experimental_noalias_scope_decl:
7221 case Intrinsic::var_annotation:
7222 case Intrinsic::sideeffect:
7223 // Discard annotate attributes, noalias scope declarations, assumptions, and
7224 // artificial side-effects.
7225 return;
7226
7227 case Intrinsic::codeview_annotation: {
7228 // Emit a label associated with this metadata.
7230 MCSymbol *Label =
7231 MF.getMMI().getContext().createTempSymbol("annotation", true);
7232 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7233 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7234 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7235 DAG.setRoot(Res);
7236 return;
7237 }
7238
7239 case Intrinsic::init_trampoline: {
7240 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7241
7242 SDValue Ops[6];
7243 Ops[0] = getRoot();
7244 Ops[1] = getValue(I.getArgOperand(0));
7245 Ops[2] = getValue(I.getArgOperand(1));
7246 Ops[3] = getValue(I.getArgOperand(2));
7247 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7248 Ops[5] = DAG.getSrcValue(F);
7249
7250 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7251
7252 DAG.setRoot(Res);
7253 return;
7254 }
7255 case Intrinsic::adjust_trampoline:
7258 getValue(I.getArgOperand(0))));
7259 return;
7260 case Intrinsic::gcroot: {
7262 "only valid in functions with gc specified, enforced by Verifier");
7263 assert(GFI && "implied by previous");
7264 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7265 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7266
7267 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7268 GFI->addStackRoot(FI->getIndex(), TypeMap);
7269 return;
7270 }
7271 case Intrinsic::gcread:
7272 case Intrinsic::gcwrite:
7273 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7274 case Intrinsic::get_rounding:
7275 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7276 setValue(&I, Res);
7277 DAG.setRoot(Res.getValue(1));
7278 return;
7279
7280 case Intrinsic::expect:
7281 // Just replace __builtin_expect(exp, c) with EXP.
7282 setValue(&I, getValue(I.getArgOperand(0)));
7283 return;
7284
7285 case Intrinsic::ubsantrap:
7286 case Intrinsic::debugtrap:
7287 case Intrinsic::trap: {
7288 StringRef TrapFuncName =
7289 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7290 if (TrapFuncName.empty()) {
7291 switch (Intrinsic) {
7292 case Intrinsic::trap:
7293 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7294 break;
7295 case Intrinsic::debugtrap:
7296 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7297 break;
7298 case Intrinsic::ubsantrap:
7300 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7302 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7303 MVT::i32)));
7304 break;
7305 default: llvm_unreachable("unknown trap intrinsic");
7306 }
7307 return;
7308 }
7310 if (Intrinsic == Intrinsic::ubsantrap) {
7312 Args[0].Val = I.getArgOperand(0);
7313 Args[0].Node = getValue(Args[0].Val);
7314 Args[0].Ty = Args[0].Val->getType();
7315 }
7316
7318 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7319 CallingConv::C, I.getType(),
7320 DAG.getExternalSymbol(TrapFuncName.data(),
7322 std::move(Args));
7323
7324 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7325 DAG.setRoot(Result.second);
7326 return;
7327 }
7328
7329 case Intrinsic::uadd_with_overflow:
7330 case Intrinsic::sadd_with_overflow:
7331 case Intrinsic::usub_with_overflow:
7332 case Intrinsic::ssub_with_overflow:
7333 case Intrinsic::umul_with_overflow:
7334 case Intrinsic::smul_with_overflow: {
7336 switch (Intrinsic) {
7337 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7338 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7339 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7340 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7341 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7342 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7343 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7344 }
7345 SDValue Op1 = getValue(I.getArgOperand(0));
7346 SDValue Op2 = getValue(I.getArgOperand(1));
7347
7348 EVT ResultVT = Op1.getValueType();
7349 EVT OverflowVT = MVT::i1;
7350 if (ResultVT.isVector())
7351 OverflowVT = EVT::getVectorVT(
7352 *Context, OverflowVT, ResultVT.getVectorElementCount());
7353
7354 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7355 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7356 return;
7357 }
7358 case Intrinsic::prefetch: {
7359 SDValue Ops[5];
7360 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7362 Ops[0] = DAG.getRoot();
7363 Ops[1] = getValue(I.getArgOperand(0));
7364 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7365 MVT::i32);
7366 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7367 MVT::i32);
7368 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7369 MVT::i32);
7371 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7372 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7373 /* align */ std::nullopt, Flags);
7374
7375 // Chain the prefetch in parallel with any pending loads, to stay out of
7376 // the way of later optimizations.
7377 PendingLoads.push_back(Result);
7378 Result = getRoot();
7379 DAG.setRoot(Result);
7380 return;
7381 }
7382 case Intrinsic::lifetime_start:
7383 case Intrinsic::lifetime_end: {
7384 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7385 // Stack coloring is not enabled in O0, discard region information.
7387 return;
7388
7389 const int64_t ObjectSize =
7390 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
7391 Value *const ObjectPtr = I.getArgOperand(1);
7393 getUnderlyingObjects(ObjectPtr, Allocas);
7394
7395 for (const Value *Alloca : Allocas) {
7396 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7397
7398 // Could not find an Alloca.
7399 if (!LifetimeObject)
7400 continue;
7401
7402 // First check that the Alloca is static, otherwise it won't have a
7403 // valid frame index.
7404 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7405 if (SI == FuncInfo.StaticAllocaMap.end())
7406 return;
7407
7408 const int FrameIndex = SI->second;
7409 int64_t Offset;
7411 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
7412 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7413 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
7414 Offset);
7415 DAG.setRoot(Res);
7416 }
7417 return;
7418 }
7419 case Intrinsic::pseudoprobe: {
7420 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7421 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7422 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7423 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7424 DAG.setRoot(Res);
7425 return;
7426 }
7427 case Intrinsic::invariant_start:
7428 // Discard region information.
7429 setValue(&I,
7430 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7431 return;
7432 case Intrinsic::invariant_end:
7433 // Discard region information.
7434 return;
7435 case Intrinsic::clear_cache:
7436 /// FunctionName may be null.
7437 if (const char *FunctionName = TLI.getClearCacheBuiltinName())
7438 lowerCallToExternalSymbol(I, FunctionName);
7439 return;
7440 case Intrinsic::donothing:
7441 case Intrinsic::seh_try_begin:
7442 case Intrinsic::seh_scope_begin:
7443 case Intrinsic::seh_try_end:
7444 case Intrinsic::seh_scope_end:
7445 // ignore
7446 return;
7447 case Intrinsic::experimental_stackmap:
7448 visitStackmap(I);
7449 return;
7450 case Intrinsic::experimental_patchpoint_void:
7451 case Intrinsic::experimental_patchpoint:
7452 visitPatchpoint(I);
7453 return;
7454 case Intrinsic::experimental_gc_statepoint:
7455 LowerStatepoint(cast<GCStatepointInst>(I));
7456 return;
7457 case Intrinsic::experimental_gc_result:
7458 visitGCResult(cast<GCResultInst>(I));
7459 return;
7460 case Intrinsic::experimental_gc_relocate:
7461 visitGCRelocate(cast<GCRelocateInst>(I));
7462 return;
7463 case Intrinsic::instrprof_cover:
7464 llvm_unreachable("instrprof failed to lower a cover");
7465 case Intrinsic::instrprof_increment:
7466 llvm_unreachable("instrprof failed to lower an increment");
7467 case Intrinsic::instrprof_timestamp:
7468 llvm_unreachable("instrprof failed to lower a timestamp");
7469 case Intrinsic::instrprof_value_profile:
7470 llvm_unreachable("instrprof failed to lower a value profiling call");
7471 case Intrinsic::instrprof_mcdc_parameters:
7472 llvm_unreachable("instrprof failed to lower mcdc parameters");
7473 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7474 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7475 case Intrinsic::instrprof_mcdc_condbitmap_update:
7476 llvm_unreachable("instrprof failed to lower an mcdc condbitmap update");
7477 case Intrinsic::localescape: {
7480
7481 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7482 // is the same on all targets.
7483 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7484 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7485 if (isa<ConstantPointerNull>(Arg))
7486 continue; // Skip null pointers. They represent a hole in index space.
7487 AllocaInst *Slot = cast<AllocaInst>(Arg);
7488 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7489 "can only escape static allocas");
7490 int FI = FuncInfo.StaticAllocaMap[Slot];
7491 MCSymbol *FrameAllocSym =
7495 TII->get(TargetOpcode::LOCAL_ESCAPE))
7496 .addSym(FrameAllocSym)
7497 .addFrameIndex(FI);
7498 }
7499
7500 return;
7501 }
7502
7503 case Intrinsic::localrecover: {
7504 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7506
7507 // Get the symbol that defines the frame offset.
7508 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7509 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7510 unsigned IdxVal =
7511 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7512 MCSymbol *FrameAllocSym =
7515
7516 Value *FP = I.getArgOperand(1);
7517 SDValue FPVal = getValue(FP);
7518 EVT PtrVT = FPVal.getValueType();
7519
7520 // Create a MCSymbol for the label to avoid any target lowering
7521 // that would make this PC relative.
7522 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7523 SDValue OffsetVal =
7524 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7525
7526 // Add the offset to the FP.
7527 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7528 setValue(&I, Add);
7529
7530 return;
7531 }
7532
7533 case Intrinsic::eh_exceptionpointer:
7534 case Intrinsic::eh_exceptioncode: {
7535 // Get the exception pointer vreg, copy from it, and resize it to fit.
7536 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7537 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7538 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7539 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7540 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7541 if (Intrinsic == Intrinsic::eh_exceptioncode)
7542 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7543 setValue(&I, N);
7544 return;
7545 }
7546 case Intrinsic::xray_customevent: {
7547 // Here we want to make sure that the intrinsic behaves as if it has a
7548 // specific calling convention.
7549 const auto &Triple = DAG.getTarget().getTargetTriple();
7550 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7551 return;
7552
7554
7555 // We want to say that we always want the arguments in registers.
7556 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7557 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7558 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7559 SDValue Chain = getRoot();
7560 Ops.push_back(LogEntryVal);
7561 Ops.push_back(StrSizeVal);
7562 Ops.push_back(Chain);
7563
7564 // We need to enforce the calling convention for the callsite, so that
7565 // argument ordering is enforced correctly, and that register allocation can
7566 // see that some registers may be assumed clobbered and have to preserve
7567 // them across calls to the intrinsic.
7568 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7569 sdl, NodeTys, Ops);
7570 SDValue patchableNode = SDValue(MN, 0);
7571 DAG.setRoot(patchableNode);
7572 setValue(&I, patchableNode);
7573 return;
7574 }
7575 case Intrinsic::xray_typedevent: {
7576 // Here we want to make sure that the intrinsic behaves as if it has a
7577 // specific calling convention.
7578 const auto &Triple = DAG.getTarget().getTargetTriple();
7579 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7580 return;
7581
7583
7584 // We want to say that we always want the arguments in registers.
7585 // It's unclear to me how manipulating the selection DAG here forces callers
7586 // to provide arguments in registers instead of on the stack.
7587 SDValue LogTypeId = getValue(I.getArgOperand(0));
7588 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7589 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7590 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7591 SDValue Chain = getRoot();
7592 Ops.push_back(LogTypeId);
7593 Ops.push_back(LogEntryVal);
7594 Ops.push_back(StrSizeVal);
7595 Ops.push_back(Chain);
7596
7597 // We need to enforce the calling convention for the callsite, so that
7598 // argument ordering is enforced correctly, and that register allocation can
7599 // see that some registers may be assumed clobbered and have to preserve
7600 // them across calls to the intrinsic.
7602 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7603 SDValue patchableNode = SDValue(MN, 0);
7604 DAG.setRoot(patchableNode);
7605 setValue(&I, patchableNode);
7606 return;
7607 }
7608 case Intrinsic::experimental_deoptimize:
7610 return;
7611 case Intrinsic::experimental_stepvector:
7612 visitStepVector(I);
7613 return;
7614 case Intrinsic::vector_reduce_fadd:
7615 case Intrinsic::vector_reduce_fmul:
7616 case Intrinsic::vector_reduce_add:
7617 case Intrinsic::vector_reduce_mul:
7618 case Intrinsic::vector_reduce_and:
7619 case Intrinsic::vector_reduce_or:
7620 case Intrinsic::vector_reduce_xor:
7621 case Intrinsic::vector_reduce_smax:
7622 case Intrinsic::vector_reduce_smin:
7623 case Intrinsic::vector_reduce_umax:
7624 case Intrinsic::vector_reduce_umin:
7625 case Intrinsic::vector_reduce_fmax:
7626 case Intrinsic::vector_reduce_fmin:
7627 case Intrinsic::vector_reduce_fmaximum:
7628 case Intrinsic::vector_reduce_fminimum:
7629 visitVectorReduce(I, Intrinsic);
7630 return;
7631
7632 case Intrinsic::icall_branch_funnel: {
7634 Ops.push_back(getValue(I.getArgOperand(0)));
7635
7636 int64_t Offset;
7637 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7638 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7639 if (!Base)
7641 "llvm.icall.branch.funnel operand must be a GlobalValue");
7642 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7643
7644 struct BranchFunnelTarget {
7645 int64_t Offset;
7647 };
7649
7650 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7651 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7652 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7653 if (ElemBase != Base)
7654 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7655 "to the same GlobalValue");
7656
7657 SDValue Val = getValue(I.getArgOperand(Op + 1));
7658 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7659 if (!GA)
7661 "llvm.icall.branch.funnel operand must be a GlobalValue");
7663 GA->getGlobal(), sdl, Val.getValueType(),
7664 GA->getOffset())});
7665 }
7666 llvm::sort(Targets,
7667 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7668 return T1.Offset < T2.Offset;
7669 });
7670
7671 for (auto &T : Targets) {
7672 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7673 Ops.push_back(T.Target);
7674 }
7675
7676 Ops.push_back(DAG.getRoot()); // Chain
7677 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7678 MVT::Other, Ops),
7679 0);
7680 DAG.setRoot(N);
7681 setValue(&I, N);
7682 HasTailCall = true;
7683 return;
7684 }
7685
7686 case Intrinsic::wasm_landingpad_index:
7687 // Information this intrinsic contained has been transferred to
7688 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7689 // delete it now.
7690 return;
7691
7692 case Intrinsic::aarch64_settag:
7693 case Intrinsic::aarch64_settag_zero: {
7695 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7697 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7698 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7699 ZeroMemory);
7700 DAG.setRoot(Val);
7701 setValue(&I, Val);
7702 return;
7703 }
7704 case Intrinsic::amdgcn_cs_chain: {
7705 assert(I.arg_size() == 5 && "Additional args not supported yet");
7706 assert(cast<ConstantInt>(I.getOperand(4))->isZero() &&
7707 "Non-zero flags not supported yet");
7708
7709 // At this point we don't care if it's amdgpu_cs_chain or
7710 // amdgpu_cs_chain_preserve.
7712
7713 Type *RetTy = I.getType();
7714 assert(RetTy->isVoidTy() && "Should not return");
7715
7716 SDValue Callee = getValue(I.getOperand(0));
7717
7718 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7719 // We'll also tack the value of the EXEC mask at the end.
7721 Args.reserve(3);
7722
7723 for (unsigned Idx : {2, 3, 1}) {
7725 Arg.Node = getValue(I.getOperand(Idx));
7726 Arg.Ty = I.getOperand(Idx)->getType();
7727 Arg.setAttributes(&I, Idx);
7728 Args.push_back(Arg);
7729 }
7730
7731 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7732 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7733 Args[2].IsInReg = true; // EXEC should be inreg
7734
7736 CLI.setDebugLoc(getCurSDLoc())
7737 .setChain(getRoot())
7738 .setCallee(CC, RetTy, Callee, std::move(Args))
7739 .setNoReturn(true)
7740 .setTailCall(true)
7741 .setConvergent(I.isConvergent());
7742 CLI.CB = &I;
7743 std::pair<SDValue, SDValue> Result =
7744 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7745 (void)Result;
7746 assert(!Result.first.getNode() && !Result.second.getNode() &&
7747 "Should've lowered as tail call");
7748
7749 HasTailCall = true;
7750 return;
7751 }
7752 case Intrinsic::ptrmask: {
7753 SDValue Ptr = getValue(I.getOperand(0));
7754 SDValue Mask = getValue(I.getOperand(1));
7755
7756 EVT PtrVT = Ptr.getValueType();
7757 assert(PtrVT == Mask.getValueType() &&
7758 "Pointers with different index type are not supported by SDAG");
7759 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
7760 return;
7761 }
7762 case Intrinsic::threadlocal_address: {
7763 setValue(&I, getValue(I.getOperand(0)));
7764 return;
7765 }
7766 case Intrinsic::get_active_lane_mask: {
7767 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7768 SDValue Index = getValue(I.getOperand(0));
7769 EVT ElementVT = Index.getValueType();
7770
7771 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7772 visitTargetIntrinsic(I, Intrinsic);
7773 return;
7774 }
7775
7776 SDValue TripCount = getValue(I.getOperand(1));
7777 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
7778 CCVT.getVectorElementCount());
7779
7780 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
7781 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
7782 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7783 SDValue VectorInduction = DAG.getNode(
7784 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7785 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7786 VectorTripCount, ISD::CondCode::SETULT);
7787 setValue(&I, SetCC);
7788 return;
7789 }
7790 case Intrinsic::experimental_get_vector_length: {
7791 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
7792 "Expected positive VF");
7793 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
7794 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
7795
7796 SDValue Count = getValue(I.getOperand(0));
7797 EVT CountVT = Count.getValueType();
7798
7799 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
7800 visitTargetIntrinsic(I, Intrinsic);
7801 return;
7802 }
7803
7804 // Expand to a umin between the trip count and the maximum elements the type
7805 // can hold.
7806 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7807
7808 // Extend the trip count to at least the result VT.
7809 if (CountVT.bitsLT(VT)) {
7810 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
7811 CountVT = VT;
7812 }
7813
7814 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
7815 ElementCount::get(VF, IsScalable));
7816
7817 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
7818 // Clip to the result type if needed.
7819 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
7820
7821 setValue(&I, Trunc);
7822 return;
7823 }
7824 case Intrinsic::experimental_cttz_elts: {
7825 auto DL = getCurSDLoc();
7826 SDValue Op = getValue(I.getOperand(0));
7827 EVT OpVT = Op.getValueType();
7828
7829 if (!TLI.shouldExpandCttzElements(OpVT)) {
7830 visitTargetIntrinsic(I, Intrinsic);
7831 return;
7832 }
7833
7834 if (OpVT.getScalarType() != MVT::i1) {
7835 // Compare the input vector elements to zero & use to count trailing zeros
7836 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
7837 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
7838 OpVT.getVectorElementCount());
7839 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
7840 }
7841
7842 // Find the smallest "sensible" element type to use for the expansion.
7843 ConstantRange CR(
7845 if (OpVT.isScalableVT())
7846 CR = CR.umul_sat(getVScaleRange(I.getCaller(), 64));
7847
7848 // If the zero-is-poison flag is set, we can assume the upper limit
7849 // of the result is VF-1.
7850 if (!cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero())
7851 CR = CR.subtract(APInt(64, 1));
7852
7853 unsigned EltWidth = I.getType()->getScalarSizeInBits();
7854 EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
7855 EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
7856
7857 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
7858
7859 // Create the new vector type & get the vector length
7860 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
7861 OpVT.getVectorElementCount());
7862
7863 SDValue VL =
7864 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
7865
7866 SDValue StepVec = DAG.getStepVector(DL, NewVT);
7867 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
7868 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
7870 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
7872 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
7873
7874 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7876
7877 setValue(&I, Ret);
7878 return;
7879 }
7880 case Intrinsic::vector_insert: {
7881 SDValue Vec = getValue(I.getOperand(0));
7882 SDValue SubVec = getValue(I.getOperand(1));
7883 SDValue Index = getValue(I.getOperand(2));
7884
7885 // The intrinsic's index type is i64, but the SDNode requires an index type
7886 // suitable for the target. Convert the index as required.
7887 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7888 if (Index.getValueType() != VectorIdxTy)
7889 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7890
7891 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7892 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7893 Index));
7894 return;
7895 }
7896 case Intrinsic::vector_extract: {
7897 SDValue Vec = getValue(I.getOperand(0));
7898 SDValue Index = getValue(I.getOperand(1));
7899 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7900
7901 // The intrinsic's index type is i64, but the SDNode requires an index type
7902 // suitable for the target. Convert the index as required.
7903 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7904 if (Index.getValueType() != VectorIdxTy)
7905 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7906
7907 setValue(&I,
7908 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7909 return;
7910 }
7911 case Intrinsic::experimental_vector_reverse:
7912 visitVectorReverse(I);
7913 return;
7914 case Intrinsic::experimental_vector_splice:
7915 visitVectorSplice(I);
7916 return;
7917 case Intrinsic::callbr_landingpad:
7918 visitCallBrLandingPad(I);
7919 return;
7920 case Intrinsic::experimental_vector_interleave2:
7921 visitVectorInterleave(I);
7922 return;
7923 case Intrinsic::experimental_vector_deinterleave2:
7924 visitVectorDeinterleave(I);
7925 return;
7926 case Intrinsic::experimental_convergence_anchor:
7927 case Intrinsic::experimental_convergence_entry:
7928 case Intrinsic::experimental_convergence_loop:
7929 visitConvergenceControl(I, Intrinsic);
7930 }
7931}
7932
7933void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7934 const ConstrainedFPIntrinsic &FPI) {
7935 SDLoc sdl = getCurSDLoc();
7936
7937 // We do not need to serialize constrained FP intrinsics against
7938 // each other or against (nonvolatile) loads, so they can be
7939 // chained like loads.
7940 SDValue Chain = DAG.getRoot();
7942 Opers.push_back(Chain);
7943 if (FPI.isUnaryOp()) {
7944 Opers.push_back(getValue(FPI.getArgOperand(0)));
7945 } else if (FPI.isTernaryOp()) {
7946 Opers.push_back(getValue(FPI.getArgOperand(0)));
7947 Opers.push_back(getValue(FPI.getArgOperand(1)));
7948 Opers.push_back(getValue(FPI.getArgOperand(2)));
7949 } else {
7950 Opers.push_back(getValue(FPI.getArgOperand(0)));
7951 Opers.push_back(getValue(FPI.getArgOperand(1)));
7952 }
7953
7954 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7955 assert(Result.getNode()->getNumValues() == 2);
7956
7957 // Push node to the appropriate list so that future instructions can be
7958 // chained up correctly.
7959 SDValue OutChain = Result.getValue(1);
7960 switch (EB) {
7962 // The only reason why ebIgnore nodes still need to be chained is that
7963 // they might depend on the current rounding mode, and therefore must
7964 // not be moved across instruction that may change that mode.
7965 [[fallthrough]];
7967 // These must not be moved across calls or instructions that may change
7968 // floating-point exception masks.
7969 PendingConstrainedFP.push_back(OutChain);
7970 break;
7972 // These must not be moved across calls or instructions that may change
7973 // floating-point exception masks or read floating-point exception flags.
7974 // In addition, they cannot be optimized out even if unused.
7975 PendingConstrainedFPStrict.push_back(OutChain);
7976 break;
7977 }
7978 };
7979
7981 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
7982 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
7984
7987 Flags.setNoFPExcept(true);
7988
7989 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7990 Flags.copyFMF(*FPOp);
7991
7992 unsigned Opcode;
7993 switch (FPI.getIntrinsicID()) {
7994 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7995#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
7996 case Intrinsic::INTRINSIC: \
7997 Opcode = ISD::STRICT_##DAGN; \
7998 break;
7999#include "llvm/IR/ConstrainedOps.def"
8000 case Intrinsic::experimental_constrained_fmuladd: {
8001 Opcode = ISD::STRICT_FMA;
8002 // Break fmuladd into fmul and fadd.
8005 Opers.pop_back();
8006 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8007 pushOutChain(Mul, EB);
8008 Opcode = ISD::STRICT_FADD;
8009 Opers.clear();
8010 Opers.push_back(Mul.getValue(1));
8011 Opers.push_back(Mul.getValue(0));
8012 Opers.push_back(getValue(FPI.getArgOperand(2)));
8013 }
8014 break;
8015 }
8016 }
8017
8018 // A few strict DAG nodes carry additional operands that are not
8019 // set up by the default code above.
8020 switch (Opcode) {
8021 default: break;
8023 Opers.push_back(
8025 break;
8026 case ISD::STRICT_FSETCC:
8027 case ISD::STRICT_FSETCCS: {
8028 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8029 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8030 if (TM.Options.NoNaNsFPMath)
8031 Condition = getFCmpCodeWithoutNaN(Condition);
8032 Opers.push_back(DAG.getCondCode(Condition));
8033 break;
8034 }
8035 }
8036
8037 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8038 pushOutChain(Result, EB);
8039
8040 SDValue FPResult = Result.getValue(0);
8041 setValue(&FPI, FPResult);
8042}
8043
8044static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8045 std::optional<unsigned> ResOPC;
8046 switch (VPIntrin.getIntrinsicID()) {
8047 case Intrinsic::vp_ctlz: {
8048 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8049 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8050 break;
8051 }
8052 case Intrinsic::vp_cttz: {
8053 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8054 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8055 break;
8056 }
8057#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8058 case Intrinsic::VPID: \
8059 ResOPC = ISD::VPSD; \
8060 break;
8061#include "llvm/IR/VPIntrinsics.def"
8062 }
8063
8064 if (!ResOPC)
8066 "Inconsistency: no SDNode available for this VPIntrinsic!");
8067
8068 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8069 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8070 if (VPIntrin.getFastMathFlags().allowReassoc())
8071 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8072 : ISD::VP_REDUCE_FMUL;
8073 }
8074
8075 return *ResOPC;
8076}
8077
8078void SelectionDAGBuilder::visitVPLoad(
8079 const VPIntrinsic &VPIntrin, EVT VT,
8080 const SmallVectorImpl<SDValue> &OpValues) {
8081 SDLoc DL = getCurSDLoc();
8082 Value *PtrOperand = VPIntrin.getArgOperand(0);
8083 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8084 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8085 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8086 SDValue LD;
8087 // Do not serialize variable-length loads of constant memory with
8088 // anything.
8089 if (!Alignment)
8090 Alignment = DAG.getEVTAlign(VT);
8091 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8092 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8093 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8096 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8097 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8098 MMO, false /*IsExpanding */);
8099 if (AddToChain)
8100 PendingLoads.push_back(LD.getValue(1));
8101 setValue(&VPIntrin, LD);
8102}
8103
8104void SelectionDAGBuilder::visitVPGather(
8105 const VPIntrinsic &VPIntrin, EVT VT,
8106 const SmallVectorImpl<SDValue> &OpValues) {
8107 SDLoc DL = getCurSDLoc();
8109 Value *PtrOperand = VPIntrin.getArgOperand(0);
8110 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8111 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8112 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8113 SDValue LD;
8114 if (!Alignment)
8115 Alignment = DAG.getEVTAlign(VT.getScalarType());
8116 unsigned AS =
8117 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8120 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8121 SDValue Base, Index, Scale;
8122 ISD::MemIndexType IndexType;
8123 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8124 this, VPIntrin.getParent(),
8125 VT.getScalarStoreSize());
8126 if (!UniformBase) {
8128 Index = getValue(PtrOperand);
8129 IndexType = ISD::SIGNED_SCALED;
8131 }
8132 EVT IdxVT = Index.getValueType();
8133 EVT EltTy = IdxVT.getVectorElementType();
8134 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8135 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8136 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8137 }
8138 LD = DAG.getGatherVP(
8139 DAG.getVTList(VT, MVT::Other), VT, DL,
8140 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8141 IndexType);
8142 PendingLoads.push_back(LD.getValue(1));
8143 setValue(&VPIntrin, LD);
8144}
8145
8146void SelectionDAGBuilder::visitVPStore(
8147 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8148 SDLoc DL = getCurSDLoc();
8149 Value *PtrOperand = VPIntrin.getArgOperand(1);
8150 EVT VT = OpValues[0].getValueType();
8151 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8152 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8153 SDValue ST;
8154 if (!Alignment)
8155 Alignment = DAG.getEVTAlign(VT);
8156 SDValue Ptr = OpValues[1];
8157 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8160 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8161 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8162 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8163 /* IsTruncating */ false, /*IsCompressing*/ false);
8164 DAG.setRoot(ST);
8165 setValue(&VPIntrin, ST);
8166}
8167
8168void SelectionDAGBuilder::visitVPScatter(
8169 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8170 SDLoc DL = getCurSDLoc();
8172 Value *PtrOperand = VPIntrin.getArgOperand(1);
8173 EVT VT = OpValues[0].getValueType();
8174 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8175 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8176 SDValue ST;
8177 if (!Alignment)
8178 Alignment = DAG.getEVTAlign(VT.getScalarType());
8179 unsigned AS =
8180 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8183 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8184 SDValue Base, Index, Scale;
8185 ISD::MemIndexType IndexType;
8186 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8187 this, VPIntrin.getParent(),
8188 VT.getScalarStoreSize());
8189 if (!UniformBase) {
8191 Index = getValue(PtrOperand);
8192 IndexType = ISD::SIGNED_SCALED;
8193 Scale =
8195 }
8196 EVT IdxVT = Index.getValueType();
8197 EVT EltTy = IdxVT.getVectorElementType();
8198 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8199 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8200 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8201 }
8202 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8203 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8204 OpValues[2], OpValues[3]},
8205 MMO, IndexType);
8206 DAG.setRoot(ST);
8207 setValue(&VPIntrin, ST);
8208}
8209
8210void SelectionDAGBuilder::visitVPStridedLoad(
8211 const VPIntrinsic &VPIntrin, EVT VT,
8212 const SmallVectorImpl<SDValue> &OpValues) {
8213 SDLoc DL = getCurSDLoc();
8214 Value *PtrOperand = VPIntrin.getArgOperand(0);
8215 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8216 if (!Alignment)
8217 Alignment = DAG.getEVTAlign(VT.getScalarType());
8218 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8219 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8220 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8221 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8222 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8223 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8226 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8227
8228 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8229 OpValues[2], OpValues[3], MMO,
8230 false /*IsExpanding*/);
8231
8232 if (AddToChain)
8233 PendingLoads.push_back(LD.getValue(1));
8234 setValue(&VPIntrin, LD);
8235}
8236
8237void SelectionDAGBuilder::visitVPStridedStore(
8238 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8239 SDLoc DL = getCurSDLoc();
8240 Value *PtrOperand = VPIntrin.getArgOperand(1);
8241 EVT VT = OpValues[0].getValueType();
8242 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8243 if (!Alignment)
8244 Alignment = DAG.getEVTAlign(VT.getScalarType());
8245 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8246 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8249 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8250
8252 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8253 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8254 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8255 /*IsCompressing*/ false);
8256
8257 DAG.setRoot(ST);
8258 setValue(&VPIntrin, ST);
8259}
8260
8261void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8263 SDLoc DL = getCurSDLoc();
8264
8265 ISD::CondCode Condition;
8267 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8268 if (IsFP) {
8269 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8270 // flags, but calls that don't return floating-point types can't be
8271 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8272 Condition = getFCmpCondCode(CondCode);
8273 if (TM.Options.NoNaNsFPMath)
8274 Condition = getFCmpCodeWithoutNaN(Condition);
8275 } else {
8276 Condition = getICmpCondCode(CondCode);
8277 }
8278
8279 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8280 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8281 // #2 is the condition code
8282 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8283 SDValue EVL = getValue(VPIntrin.getOperand(4));
8284 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8285 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8286 "Unexpected target EVL type");
8287 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8288
8290 VPIntrin.getType());
8291 setValue(&VPIntrin,
8292 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8293}
8294
8295void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8296 const VPIntrinsic &VPIntrin) {
8297 SDLoc DL = getCurSDLoc();
8298 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8299
8300 auto IID = VPIntrin.getIntrinsicID();
8301
8302 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8303 return visitVPCmp(*CmpI);
8304
8305 SmallVector<EVT, 4> ValueVTs;
8307 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8308 SDVTList VTs = DAG.getVTList(ValueVTs);
8309
8310 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8311
8312 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8313 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8314 "Unexpected target EVL type");
8315
8316 // Request operands.
8317 SmallVector<SDValue, 7> OpValues;
8318 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8319 auto Op = getValue(VPIntrin.getArgOperand(I));
8320 if (I == EVLParamPos)
8321 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8322 OpValues.push_back(Op);
8323 }
8324
8325 switch (Opcode) {
8326 default: {
8327 SDNodeFlags SDFlags;
8328 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8329 SDFlags.copyFMF(*FPMO);
8330 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8331 setValue(&VPIntrin, Result);
8332 break;
8333 }
8334 case ISD::VP_LOAD:
8335 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8336 break;
8337 case ISD::VP_GATHER:
8338 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8339 break;
8340 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8341 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8342 break;
8343 case ISD::VP_STORE:
8344 visitVPStore(VPIntrin, OpValues);
8345 break;
8346 case ISD::VP_SCATTER:
8347 visitVPScatter(VPIntrin, OpValues);
8348 break;
8349 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8350 visitVPStridedStore(VPIntrin, OpValues);
8351 break;
8352 case ISD::VP_FMULADD: {
8353 assert(OpValues.size() == 5 && "Unexpected number of operands");
8354 SDNodeFlags SDFlags;
8355 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8356 SDFlags.copyFMF(*FPMO);
8359 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8360 } else {
8362 ISD::VP_FMUL, DL, VTs,
8363 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8364 SDValue Add =
8365 DAG.getNode(ISD::VP_FADD, DL, VTs,
8366 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8367 setValue(&VPIntrin, Add);
8368 }
8369 break;
8370 }
8371 case ISD::VP_IS_FPCLASS: {
8372 const DataLayout DLayout = DAG.getDataLayout();
8373 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8374 auto Constant = OpValues[1]->getAsZExtVal();
8376 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8377 {OpValues[0], Check, OpValues[2], OpValues[3]});
8378 setValue(&VPIntrin, V);
8379 return;
8380 }
8381 case ISD::VP_INTTOPTR: {
8382 SDValue N = OpValues[0];
8383 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8384 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8385 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8386 OpValues[2]);
8387 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8388 OpValues[2]);
8389 setValue(&VPIntrin, N);
8390 break;
8391 }
8392 case ISD::VP_PTRTOINT: {
8393 SDValue N = OpValues[0];
8395 VPIntrin.getType());
8396 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8397 VPIntrin.getOperand(0)->getType());
8398 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8399 OpValues[2]);
8400 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8401 OpValues[2]);
8402 setValue(&VPIntrin, N);
8403 break;
8404 }
8405 case ISD::VP_ABS:
8406 case ISD::VP_CTLZ:
8407 case ISD::VP_CTLZ_ZERO_UNDEF:
8408 case ISD::VP_CTTZ:
8409 case ISD::VP_CTTZ_ZERO_UNDEF: {
8410 SDValue Result =
8411 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8412 setValue(&VPIntrin, Result);
8413 break;
8414 }
8415 }
8416}
8417
8418SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8419 const BasicBlock *EHPadBB,
8420 MCSymbol *&BeginLabel) {
8422 MachineModuleInfo &MMI = MF.getMMI();
8423
8424 // Insert a label before the invoke call to mark the try range. This can be
8425 // used to detect deletion of the invoke via the MachineModuleInfo.
8426 BeginLabel = MMI.getContext().createTempSymbol();
8427
8428 // For SjLj, keep track of which landing pads go with which invokes
8429 // so as to maintain the ordering of pads in the LSDA.
8430 unsigned CallSiteIndex = MMI.getCurrentCallSite();
8431 if (CallSiteIndex) {
8432 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8433 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
8434
8435 // Now that the call site is handled, stop tracking it.
8436 MMI.setCurrentCallSite(0);
8437 }
8438
8439 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8440}
8441
8442SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8443 const BasicBlock *EHPadBB,
8444 MCSymbol *BeginLabel) {
8445 assert(BeginLabel && "BeginLabel should've been set");
8446
8448 MachineModuleInfo &MMI = MF.getMMI();
8449
8450 // Insert a label at the end of the invoke call to mark the try range. This
8451 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8452 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
8453 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8454
8455 // Inform MachineModuleInfo of range.
8457 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8458 // actually use outlined funclets and their LSDA info style.
8459 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8460 assert(II && "II should've been set");
8461 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8462 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8463 } else if (!isScopedEHPersonality(Pers)) {
8464 assert(EHPadBB);
8465 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
8466 }
8467
8468 return Chain;
8469}
8470
8471std::pair<SDValue, SDValue>
8473 const BasicBlock *EHPadBB) {
8474 MCSymbol *BeginLabel = nullptr;
8475
8476 if (EHPadBB) {
8477 // Both PendingLoads and PendingExports must be flushed here;
8478 // this call might not return.
8479 (void)getRoot();
8480 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8481 CLI.setChain(getRoot());
8482 }
8483
8485 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8486
8487 assert((CLI.IsTailCall || Result.second.getNode()) &&
8488 "Non-null chain expected with non-tail call!");
8489 assert((Result.second.getNode() || !Result.first.getNode()) &&
8490 "Null value expected with tail call!");
8491
8492 if (!Result.second.getNode()) {
8493 // As a special case, a null chain means that a tail call has been emitted
8494 // and the DAG root is already updated.
8495 HasTailCall = true;
8496
8497 // Since there's no actual continuation from this block, nothing can be
8498 // relying on us setting vregs for them.
8499 PendingExports.clear();
8500 } else {
8501 DAG.setRoot(Result.second);
8502 }
8503
8504 if (EHPadBB) {
8505 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8506 BeginLabel));
8507 }
8508
8509 return Result;
8510}
8511
8513 bool isTailCall,
8514 bool isMustTailCall,
8515 const BasicBlock *EHPadBB) {
8516 auto &DL = DAG.getDataLayout();
8517 FunctionType *FTy = CB.getFunctionType();
8518 Type *RetTy = CB.getType();
8519
8521 Args.reserve(CB.arg_size());
8522
8523 const Value *SwiftErrorVal = nullptr;
8525
8526 if (isTailCall) {
8527 // Avoid emitting tail calls in functions with the disable-tail-calls
8528 // attribute.
8529 auto *Caller = CB.getParent()->getParent();
8530 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8531 "true" && !isMustTailCall)
8532 isTailCall = false;
8533
8534 // We can't tail call inside a function with a swifterror argument. Lowering
8535 // does not support this yet. It would have to move into the swifterror
8536 // register before the call.
8537 if (TLI.supportSwiftError() &&
8538 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8539 isTailCall = false;
8540 }
8541
8542 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8544 const Value *V = *I;
8545
8546 // Skip empty types
8547 if (V->getType()->isEmptyTy())
8548 continue;
8549
8550 SDValue ArgNode = getValue(V);
8551 Entry.Node = ArgNode; Entry.Ty = V->getType();
8552
8553 Entry.setAttributes(&CB, I - CB.arg_begin());
8554
8555 // Use swifterror virtual register as input to the call.
8556 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8557 SwiftErrorVal = V;
8558 // We find the virtual register for the actual swifterror argument.
8559 // Instead of using the Value, we use the virtual register instead.
8560 Entry.Node =
8562 EVT(TLI.getPointerTy(DL)));
8563 }
8564
8565 Args.push_back(Entry);
8566
8567 // If we have an explicit sret argument that is an Instruction, (i.e., it
8568 // might point to function-local memory), we can't meaningfully tail-call.
8569 if (Entry.IsSRet && isa<Instruction>(V))
8570 isTailCall = false;
8571 }
8572
8573 // If call site has a cfguardtarget operand bundle, create and add an
8574 // additional ArgListEntry.
8575 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8577 Value *V = Bundle->Inputs[0];
8578 SDValue ArgNode = getValue(V);
8579 Entry.Node = ArgNode;
8580 Entry.Ty = V->getType();
8581 Entry.IsCFGuardTarget = true;
8582 Args.push_back(Entry);
8583 }
8584
8585 // Check if target-independent constraints permit a tail call here.
8586 // Target-dependent constraints are checked within TLI->LowerCallTo.
8587 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
8588 isTailCall = false;
8589
8590 // Disable tail calls if there is an swifterror argument. Targets have not
8591 // been updated to support tail calls.
8592 if (TLI.supportSwiftError() && SwiftErrorVal)
8593 isTailCall = false;
8594
8595 ConstantInt *CFIType = nullptr;
8596 if (CB.isIndirectCall()) {
8597 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8598 if (!TLI.supportKCFIBundles())
8600 "Target doesn't support calls with kcfi operand bundles.");
8601 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8602 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8603 }
8604 }
8605
8606 SDValue ConvControlToken;
8607 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8608 auto *Token = Bundle->Inputs[0].get();
8609 ConvControlToken = getValue(Token);
8610 } else {
8611 ConvControlToken = DAG.getUNDEF(MVT::Untyped);
8612 }
8613
8616 .setChain(getRoot())
8617 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8618 .setTailCall(isTailCall)
8622 .setCFIType(CFIType)
8623 .setConvergenceControlToken(ConvControlToken);
8624 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8625
8626 if (Result.first.getNode()) {
8627 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8628 setValue(&CB, Result.first);
8629 }
8630
8631 // The last element of CLI.InVals has the SDValue for swifterror return.
8632 // Here we copy it to a virtual register and update SwiftErrorMap for
8633 // book-keeping.
8634 if (SwiftErrorVal && TLI.supportSwiftError()) {
8635 // Get the last element of InVals.
8636 SDValue Src = CLI.InVals.back();
8637 Register VReg =
8638 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8639 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8640 DAG.setRoot(CopyNode);
8641 }
8642}
8643
8644static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8645 SelectionDAGBuilder &Builder) {
8646 // Check to see if this load can be trivially constant folded, e.g. if the
8647 // input is from a string literal.
8648 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
8649 // Cast pointer to the type we really want to load.
8650 Type *LoadTy =
8651 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
8652 if (LoadVT.isVector())
8653 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
8654
8655 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
8656 PointerType::getUnqual(LoadTy));
8657
8658 if (const Constant *LoadCst =
8659 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
8660 LoadTy, Builder.DAG.getDataLayout()))
8661 return Builder.getValue(LoadCst);
8662 }
8663
8664 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
8665 // still constant memory, the input chain can be the entry node.
8666 SDValue Root;
8667 bool ConstantMemory = false;
8668
8669 // Do not serialize (non-volatile) loads of constant memory with anything.
8670 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
8671 Root = Builder.DAG.getEntryNode();
8672 ConstantMemory = true;
8673 } else {
8674 // Do not serialize non-volatile loads against each other.
8675 Root = Builder.DAG.getRoot();
8676 }
8677
8678 SDValue Ptr = Builder.getValue(PtrVal);
8679 SDValue LoadVal =
8680 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
8681 MachinePointerInfo(PtrVal), Align(1));
8682
8683 if (!ConstantMemory)
8684 Builder.PendingLoads.push_back(LoadVal.getValue(1));
8685 return LoadVal;
8686}
8687
8688/// Record the value for an instruction that produces an integer result,
8689/// converting the type where necessary.
8690void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
8691 SDValue Value,
8692 bool IsSigned) {
8694 I.getType(), true);
8695 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
8696 setValue(&I, Value);
8697}
8698
8699/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8700/// true and lower it. Otherwise return false, and it will be lowered like a
8701/// normal call.
8702/// The caller already checked that \p I calls the appropriate LibFunc with a
8703/// correct prototype.
8704bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
8705 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
8706 const Value *Size = I.getArgOperand(2);
8707 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
8708 if (CSize && CSize->getZExtValue() == 0) {
8710 I.getType(), true);
8711 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
8712 return true;
8713 }
8714
8716 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
8717 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
8719 if (Res.first.getNode()) {
8720 processIntegerCallValue(I, Res.first, true);
8721 PendingLoads.push_back(Res.second);
8722 return true;
8723 }
8724
8725 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
8726 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
8727 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
8728 return false;
8729
8730 // If the target has a fast compare for the given size, it will return a
8731 // preferred load type for that size. Require that the load VT is legal and
8732 // that the target supports unaligned loads of that type. Otherwise, return
8733 // INVALID.
8734 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
8736 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
8737 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
8738 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
8739 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
8740 // TODO: Check alignment of src and dest ptrs.
8741 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
8742 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
8743 if (!TLI.isTypeLegal(LVT) ||
8744 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
8745 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
8747 }
8748
8749 return LVT;
8750 };
8751
8752 // This turns into unaligned loads. We only do this if the target natively
8753 // supports the MVT we'll be loading or if it is small enough (<= 4) that
8754 // we'll only produce a small number of byte loads.
8755 MVT LoadVT;
8756 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
8757 switch (NumBitsToCompare) {
8758 default:
8759 return false;
8760 case 16:
8761 LoadVT = MVT::i16;
8762 break;
8763 case 32:
8764 LoadVT = MVT::i32;
8765 break;
8766 case 64:
8767 case 128:
8768 case 256:
8769 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
8770 break;
8771 }
8772
8773 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
8774 return false;
8775
8776 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
8777 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
8778
8779 // Bitcast to a wide integer type if the loads are vectors.
8780 if (LoadVT.isVector()) {
8781 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
8782 LoadL = DAG.getBitcast(CmpVT, LoadL);
8783 LoadR = DAG.getBitcast(CmpVT, LoadR);
8784 }
8785
8786 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
8787 processIntegerCallValue(I, Cmp, false);
8788 return true;
8789}
8790
8791/// See if we can lower a memchr call into an optimized form. If so, return
8792/// true and lower it. Otherwise return false, and it will be lowered like a
8793/// normal call.
8794/// The caller already checked that \p I calls the appropriate LibFunc with a
8795/// correct prototype.
8796bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8797 const Value *Src = I.getArgOperand(0);
8798 const Value *Char = I.getArgOperand(1);
8799 const Value *Length = I.getArgOperand(2);
8800
8802 std::pair<SDValue, SDValue> Res =
8804 getValue(Src), getValue(Char), getValue(Length),
8805 MachinePointerInfo(Src));
8806 if (Res.first.getNode()) {
8807 setValue(&I, Res.first);
8808 PendingLoads.push_back(Res.second);
8809 return true;
8810 }
8811
8812 return false;
8813}
8814
8815/// See if we can lower a mempcpy call into an optimized form. If so, return
8816/// true and lower it. Otherwise return false, and it will be lowered like a
8817/// normal call.
8818/// The caller already checked that \p I calls the appropriate LibFunc with a
8819/// correct prototype.
8820bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8821 SDValue Dst = getValue(I.getArgOperand(0));
8822 SDValue Src = getValue(I.getArgOperand(1));
8823 SDValue Size = getValue(I.getArgOperand(2));
8824
8825 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8826 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8827 // DAG::getMemcpy needs Alignment to be defined.
8828 Align Alignment = std::min(DstAlign, SrcAlign);
8829
8830 SDLoc sdl = getCurSDLoc();
8831
8832 // In the mempcpy context we need to pass in a false value for isTailCall
8833 // because the return pointer needs to be adjusted by the size of
8834 // the copied memory.
8835 SDValue Root = getMemoryRoot();
8836 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
8837 /*isTailCall=*/false,
8838 MachinePointerInfo(I.getArgOperand(0)),
8839 MachinePointerInfo(I.getArgOperand(1)),
8840 I.getAAMetadata());
8841 assert(MC.getNode() != nullptr &&
8842 "** memcpy should not be lowered as TailCall in mempcpy context **");
8843 DAG.setRoot(MC);
8844
8845 // Check if Size needs to be truncated or extended.
8846 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8847
8848 // Adjust return pointer to point just past the last dst byte.
8849 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8850 Dst, Size);
8851 setValue(&I, DstPlusSize);
8852 return true;
8853}
8854
8855/// See if we can lower a strcpy call into an optimized form. If so, return
8856/// true and lower it, otherwise return false and it will be lowered like a
8857/// normal call.
8858/// The caller already checked that \p I calls the appropriate LibFunc with a
8859/// correct prototype.
8860bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8861 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8862
8864 std::pair<SDValue, SDValue> Res =
8866 getValue(Arg0), getValue(Arg1),
8867 MachinePointerInfo(Arg0),
8868 MachinePointerInfo(Arg1), isStpcpy);
8869 if (Res.first.getNode()) {
8870 setValue(&I, Res.first);
8871 DAG.setRoot(Res.second);
8872 return true;
8873 }
8874
8875 return false;
8876}
8877
8878/// See if we can lower a strcmp call into an optimized form. If so, return
8879/// true and lower it, otherwise return false and it will be lowered like a
8880/// normal call.
8881/// The caller already checked that \p I calls the appropriate LibFunc with a
8882/// correct prototype.
8883bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8884 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8885
8887 std::pair<SDValue, SDValue> Res =
8889 getValue(Arg0), getValue(Arg1),
8890 MachinePointerInfo(Arg0),
8891 MachinePointerInfo(Arg1));
8892 if (Res.first.getNode()) {
8893 processIntegerCallValue(I, Res.first, true);
8894 PendingLoads.push_back(Res.second);
8895 return true;
8896 }
8897
8898 return false;
8899}
8900
8901/// See if we can lower a strlen call into an optimized form. If so, return
8902/// true and lower it, otherwise return false and it will be lowered like a
8903/// normal call.
8904/// The caller already checked that \p I calls the appropriate LibFunc with a
8905/// correct prototype.
8906bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8907 const Value *Arg0 = I.getArgOperand(0);
8908
8910 std::pair<SDValue, SDValue> Res =
8912 getValue(Arg0), MachinePointerInfo(Arg0));
8913 if (Res.first.getNode()) {
8914 processIntegerCallValue(I, Res.first, false);
8915 PendingLoads.push_back(Res.second);
8916 return true;
8917 }
8918
8919 return false;
8920}
8921
8922/// See if we can lower a strnlen call into an optimized form. If so, return
8923/// true and lower it, otherwise return false and it will be lowered like a
8924/// normal call.
8925/// The caller already checked that \p I calls the appropriate LibFunc with a
8926/// correct prototype.
8927bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8928 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8929
8931 std::pair<SDValue, SDValue> Res =
8933 getValue(Arg0), getValue(Arg1),
8934 MachinePointerInfo(Arg0));
8935 if (Res.first.getNode()) {
8936 processIntegerCallValue(I, Res.first, false);
8937 PendingLoads.push_back(Res.second);
8938 return true;
8939 }
8940
8941 return false;
8942}
8943
8944/// See if we can lower a unary floating-point operation into an SDNode with
8945/// the specified Opcode. If so, return true and lower it, otherwise return
8946/// false and it will be lowered like a normal call.
8947/// The caller already checked that \p I calls the appropriate LibFunc with a
8948/// correct prototype.
8949bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8950 unsigned Opcode) {
8951 // We already checked this call's prototype; verify it doesn't modify errno.
8952 if (!I.onlyReadsMemory())
8953 return false;
8954
8956 Flags.copyFMF(cast<FPMathOperator>(I));
8957
8958 SDValue Tmp = getValue(I.getArgOperand(0));
8959 setValue(&I,
8960 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8961 return true;
8962}
8963
8964/// See if we can lower a binary floating-point operation into an SDNode with
8965/// the specified Opcode. If so, return true and lower it. Otherwise return
8966/// false, and it will be lowered like a normal call.
8967/// The caller already checked that \p I calls the appropriate LibFunc with a
8968/// correct prototype.
8969bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8970 unsigned Opcode) {
8971 // We already checked this call's prototype; verify it doesn't modify errno.
8972 if (!I.onlyReadsMemory())
8973 return false;
8974
8976 Flags.copyFMF(cast<FPMathOperator>(I));
8977
8978 SDValue Tmp0 = getValue(I.getArgOperand(0));
8979 SDValue Tmp1 = getValue(I.getArgOperand(1));
8980 EVT VT = Tmp0.getValueType();
8981 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8982 return true;
8983}
8984
8985void SelectionDAGBuilder::visitCall(const CallInst &I) {
8986 // Handle inline assembly differently.
8987 if (I.isInlineAsm()) {
8988 visitInlineAsm(I);
8989 return;
8990 }
8991
8993
8994 if (Function *F = I.getCalledFunction()) {
8995 if (F->isDeclaration()) {
8996 // Is this an LLVM intrinsic or a target-specific intrinsic?
8997 unsigned IID = F->getIntrinsicID();
8998 if (!IID)
8999 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
9000 IID = II->getIntrinsicID(F);
9001
9002 if (IID) {
9003 visitIntrinsicCall(I, IID);
9004 return;
9005 }
9006 }
9007
9008 // Check for well-known libc/libm calls. If the function is internal, it
9009 // can't be a library call. Don't do the check if marked as nobuiltin for
9010 // some reason or the call site requires strict floating point semantics.
9011 LibFunc Func;
9012 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9013 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9015 switch (Func) {
9016 default: break;
9017 case LibFunc_bcmp:
9018 if (visitMemCmpBCmpCall(I))
9019 return;
9020 break;
9021 case LibFunc_copysign:
9022 case LibFunc_copysignf:
9023 case LibFunc_copysignl:
9024 // We already checked this call's prototype; verify it doesn't modify
9025 // errno.
9026 if (I.onlyReadsMemory()) {
9027 SDValue LHS = getValue(I.getArgOperand(0));
9028 SDValue RHS = getValue(I.getArgOperand(1));
9030 LHS.getValueType(), LHS, RHS));
9031 return;
9032 }
9033 break;
9034 case LibFunc_fabs:
9035 case LibFunc_fabsf:
9036 case LibFunc_fabsl:
9037 if (visitUnaryFloatCall(I, ISD::FABS))
9038 return;
9039 break;
9040 case LibFunc_fmin:
9041 case LibFunc_fminf:
9042 case LibFunc_fminl:
9043 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9044 return;
9045 break;
9046 case LibFunc_fmax:
9047 case LibFunc_fmaxf:
9048 case LibFunc_fmaxl:
9049 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9050 return;
9051 break;
9052 case LibFunc_sin:
9053 case LibFunc_sinf:
9054 case LibFunc_sinl:
9055 if (visitUnaryFloatCall(I, ISD::FSIN))
9056 return;
9057 break;
9058 case LibFunc_cos:
9059 case LibFunc_cosf:
9060 case LibFunc_cosl:
9061 if (visitUnaryFloatCall(I, ISD::FCOS))
9062 return;
9063 break;
9064 case LibFunc_sqrt:
9065 case LibFunc_sqrtf:
9066 case LibFunc_sqrtl:
9067 case LibFunc_sqrt_finite:
9068 case LibFunc_sqrtf_finite:
9069 case LibFunc_sqrtl_finite:
9070 if (visitUnaryFloatCall(I, ISD::FSQRT))
9071 return;
9072 break;
9073 case LibFunc_floor:
9074 case LibFunc_floorf:
9075 case LibFunc_floorl:
9076 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9077 return;
9078 break;
9079 case LibFunc_nearbyint:
9080 case LibFunc_nearbyintf:
9081 case LibFunc_nearbyintl:
9082 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9083 return;
9084 break;
9085 case LibFunc_ceil:
9086 case LibFunc_ceilf:
9087 case LibFunc_ceill:
9088 if (visitUnaryFloatCall(I, ISD::FCEIL))
9089 return;
9090 break;
9091 case LibFunc_rint:
9092 case LibFunc_rintf:
9093 case LibFunc_rintl:
9094 if (visitUnaryFloatCall(I, ISD::FRINT))
9095 return;
9096 break;
9097 case LibFunc_round:
9098 case LibFunc_roundf:
9099 case LibFunc_roundl:
9100 if (visitUnaryFloatCall(I, ISD::FROUND))
9101 return;
9102 break;
9103 case LibFunc_trunc:
9104 case LibFunc_truncf:
9105 case LibFunc_truncl:
9106 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9107 return;
9108 break;
9109 case LibFunc_log2:
9110 case LibFunc_log2f:
9111 case LibFunc_log2l:
9112 if (visitUnaryFloatCall(I, ISD::FLOG2))
9113 return;
9114 break;
9115 case LibFunc_exp2:
9116 case LibFunc_exp2f:
9117 case LibFunc_exp2l:
9118 if (visitUnaryFloatCall(I, ISD::FEXP2))
9119 return;
9120 break;
9121 case LibFunc_exp10:
9122 case LibFunc_exp10f:
9123 case LibFunc_exp10l:
9124 if (visitUnaryFloatCall(I, ISD::FEXP10))
9125 return;
9126 break;
9127 case LibFunc_ldexp:
9128 case LibFunc_ldexpf:
9129 case LibFunc_ldexpl:
9130 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9131 return;
9132 break;
9133 case LibFunc_memcmp:
9134 if (visitMemCmpBCmpCall(I))
9135 return;
9136 break;
9137 case LibFunc_mempcpy:
9138 if (visitMemPCpyCall(I))
9139 return;
9140 break;
9141 case LibFunc_memchr:
9142 if (visitMemChrCall(I))
9143 return;
9144 break;
9145 case LibFunc_strcpy:
9146 if (visitStrCpyCall(I, false))
9147 return;
9148 break;
9149 case LibFunc_stpcpy:
9150 if (visitStrCpyCall(I, true))
9151 return;
9152 break;
9153 case LibFunc_strcmp:
9154 if (visitStrCmpCall(I))
9155 return;
9156 break;
9157 case LibFunc_strlen:
9158 if (visitStrLenCall(I))
9159 return;
9160 break;
9161 case LibFunc_strnlen:
9162 if (visitStrNLenCall(I))
9163 return;
9164 break;
9165 }
9166 }
9167 }
9168
9169 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9170 // have to do anything here to lower funclet bundles.
9171 // CFGuardTarget bundles are lowered in LowerCallTo.
9172 assert(!I.hasOperandBundlesOtherThan(
9173 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9174 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9175 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9176 LLVMContext::OB_convergencectrl}) &&
9177 "Cannot lower calls with arbitrary operand bundles!");
9178
9179 SDValue Callee = getValue(I.getCalledOperand());
9180
9181 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
9182 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9183 else
9184 // Check if we can potentially perform a tail call. More detailed checking
9185 // is be done within LowerCallTo, after more information about the call is
9186 // known.
9187 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9188}
9189
9190namespace {
9191
9192/// AsmOperandInfo - This contains information for each constraint that we are
9193/// lowering.
9194class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9195public:
9196 /// CallOperand - If this is the result output operand or a clobber
9197 /// this is null, otherwise it is the incoming operand to the CallInst.
9198 /// This gets modified as the asm is processed.
9199 SDValue CallOperand;
9200
9201 /// AssignedRegs - If this is a register or register class operand, this
9202 /// contains the set of register corresponding to the operand.
9203 RegsForValue AssignedRegs;
9204
9205 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9206 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9207 }
9208
9209 /// Whether or not this operand accesses memory
9210 bool hasMemory(const TargetLowering &TLI) const {
9211 // Indirect operand accesses access memory.
9212 if (isIndirect)
9213 return true;
9214
9215 for (const auto &Code : Codes)
9217 return true;
9218
9219 return false;
9220 }
9221};
9222
9223
9224} // end anonymous namespace
9225
9226/// Make sure that the output operand \p OpInfo and its corresponding input
9227/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9228/// out).
9229static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9230 SDISelAsmOperandInfo &MatchingOpInfo,
9231 SelectionDAG &DAG) {
9232 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9233 return;
9234
9236 const auto &TLI = DAG.getTargetLoweringInfo();
9237
9238 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9239 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9240 OpInfo.ConstraintVT);
9241 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9242 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9243 MatchingOpInfo.ConstraintVT);
9244 if ((OpInfo.ConstraintVT.isInteger() !=
9245 MatchingOpInfo.ConstraintVT.isInteger()) ||
9246 (MatchRC.second != InputRC.second)) {
9247 // FIXME: error out in a more elegant fashion
9248 report_fatal_error("Unsupported asm: input constraint"
9249 " with a matching output constraint of"
9250 " incompatible type!");
9251 }
9252 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9253}
9254
9255/// Get a direct memory input to behave well as an indirect operand.
9256/// This may introduce stores, hence the need for a \p Chain.
9257/// \return The (possibly updated) chain.
9258static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9259 SDISelAsmOperandInfo &OpInfo,
9260 SelectionDAG &DAG) {
9261 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9262
9263 // If we don't have an indirect input, put it in the constpool if we can,
9264 // otherwise spill it to a stack slot.
9265 // TODO: This isn't quite right. We need to handle these according to
9266 // the addressing mode that the constraint wants. Also, this may take
9267 // an additional register for the computation and we don't want that
9268 // either.
9269
9270 // If the operand is a float, integer, or vector constant, spill to a
9271 // constant pool entry to get its address.
9272 const Value *OpVal = OpInfo.CallOperandVal;
9273 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9274 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9275 OpInfo.CallOperand = DAG.getConstantPool(
9276 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9277 return Chain;
9278 }
9279
9280 // Otherwise, create a stack slot and emit a store to it before the asm.
9281 Type *Ty = OpVal->getType();
9282 auto &DL = DAG.getDataLayout();
9283 uint64_t TySize = DL.getTypeAllocSize(Ty);
9285 int SSFI = MF.getFrameInfo().CreateStackObject(
9286 TySize, DL.getPrefTypeAlign(Ty), false);
9287 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9288 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9290 TLI.getMemValueType(DL, Ty));
9291 OpInfo.CallOperand = StackSlot;
9292
9293 return Chain;
9294}
9295
9296/// GetRegistersForValue - Assign registers (virtual or physical) for the
9297/// specified operand. We prefer to assign virtual registers, to allow the
9298/// register allocator to handle the assignment process. However, if the asm
9299/// uses features that we can't model on machineinstrs, we have SDISel do the
9300/// allocation. This produces generally horrible, but correct, code.
9301///
9302/// OpInfo describes the operand
9303/// RefOpInfo describes the matching operand if any, the operand otherwise
9304static std::optional<unsigned>
9306 SDISelAsmOperandInfo &OpInfo,
9307 SDISelAsmOperandInfo &RefOpInfo) {
9308 LLVMContext &Context = *DAG.getContext();
9309 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9310
9314
9315 // No work to do for memory/address operands.
9316 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9317 OpInfo.ConstraintType == TargetLowering::C_Address)
9318 return std::nullopt;
9319
9320 // If this is a constraint for a single physreg, or a constraint for a
9321 // register class, find it.
9322 unsigned AssignedReg;
9323 const TargetRegisterClass *RC;
9324 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9325 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9326 // RC is unset only on failure. Return immediately.
9327 if (!RC)
9328 return std::nullopt;
9329
9330 // Get the actual register value type. This is important, because the user
9331 // may have asked for (e.g.) the AX register in i32 type. We need to
9332 // remember that AX is actually i16 to get the right extension.
9333 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9334
9335 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9336 // If this is an FP operand in an integer register (or visa versa), or more
9337 // generally if the operand value disagrees with the register class we plan
9338 // to stick it in, fix the operand type.
9339 //
9340 // If this is an input value, the bitcast to the new type is done now.
9341 // Bitcast for output value is done at the end of visitInlineAsm().
9342 if ((OpInfo.Type == InlineAsm::isOutput ||
9343 OpInfo.Type == InlineAsm::isInput) &&
9344 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9345 // Try to convert to the first EVT that the reg class contains. If the
9346 // types are identical size, use a bitcast to convert (e.g. two differing
9347 // vector types). Note: output bitcast is done at the end of
9348 // visitInlineAsm().
9349 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9350 // Exclude indirect inputs while they are unsupported because the code
9351 // to perform the load is missing and thus OpInfo.CallOperand still
9352 // refers to the input address rather than the pointed-to value.
9353 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9354 OpInfo.CallOperand =
9355 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9356 OpInfo.ConstraintVT = RegVT;
9357 // If the operand is an FP value and we want it in integer registers,
9358 // use the corresponding integer type. This turns an f64 value into
9359 // i64, which can be passed with two i32 values on a 32-bit machine.
9360 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9361 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9362 if (OpInfo.Type == InlineAsm::isInput)
9363 OpInfo.CallOperand =
9364 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9365 OpInfo.ConstraintVT = VT;
9366 }
9367 }
9368 }
9369
9370 // No need to allocate a matching input constraint since the constraint it's
9371 // matching to has already been allocated.
9372 if (OpInfo.isMatchingInputConstraint())
9373 return std::nullopt;
9374
9375 EVT ValueVT = OpInfo.ConstraintVT;
9376 if (OpInfo.ConstraintVT == MVT::Other)
9377 ValueVT = RegVT;
9378
9379 // Initialize NumRegs.
9380 unsigned NumRegs = 1;
9381 if (OpInfo.ConstraintVT != MVT::Other)
9382 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9383
9384 // If this is a constraint for a specific physical register, like {r17},
9385 // assign it now.
9386
9387 // If this associated to a specific register, initialize iterator to correct
9388 // place. If virtual, make sure we have enough registers
9389
9390 // Initialize iterator if necessary
9393
9394 // Do not check for single registers.
9395 if (AssignedReg) {
9396 I = std::find(I, RC->end(), AssignedReg);
9397 if (I == RC->end()) {
9398 // RC does not contain the selected register, which indicates a
9399 // mismatch between the register and the required type/bitwidth.
9400 return {AssignedReg};
9401 }
9402 }
9403
9404 for (; NumRegs; --NumRegs, ++I) {
9405 assert(I != RC->end() && "Ran out of registers to allocate!");
9406 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9407 Regs.push_back(R);
9408 }
9409
9410 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9411 return std::nullopt;
9412}
9413
9414static unsigned
9416 const std::vector<SDValue> &AsmNodeOperands) {
9417 // Scan until we find the definition we already emitted of this operand.
9418 unsigned CurOp = InlineAsm::Op_FirstOperand;
9419 for (; OperandNo; --OperandNo) {
9420 // Advance to the next operand.
9421 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9422 const InlineAsm::Flag F(OpFlag);
9423 assert(
9424 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9425 "Skipped past definitions?");
9426 CurOp += F.getNumOperandRegisters() + 1;
9427 }
9428 return CurOp;
9429}
9430
9431namespace {
9432
9433class ExtraFlags {
9434 unsigned Flags = 0;
9435
9436public:
9437 explicit ExtraFlags(const CallBase &Call) {
9438 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9439 if (IA->hasSideEffects())
9441 if (IA->isAlignStack())
9443 if (Call.isConvergent())
9445 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9446 }
9447
9448 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9449 // Ideally, we would only check against memory constraints. However, the
9450 // meaning of an Other constraint can be target-specific and we can't easily
9451 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9452 // for Other constraints as well.
9455 if (OpInfo.Type == InlineAsm::isInput)
9457 else if (OpInfo.Type == InlineAsm::isOutput)
9459 else if (OpInfo.Type == InlineAsm::isClobber)
9461 }
9462 }
9463
9464 unsigned get() const { return Flags; }
9465};
9466
9467} // end anonymous namespace
9468
9469static bool isFunction(SDValue Op) {
9470 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9471 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9472 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9473
9474 // In normal "call dllimport func" instruction (non-inlineasm) it force
9475 // indirect access by specifing call opcode. And usually specially print
9476 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9477 // not do in this way now. (In fact, this is similar with "Data Access"
9478 // action). So here we ignore dllimport function.
9479 if (Fn && !Fn->hasDLLImportStorageClass())
9480 return true;
9481 }
9482 }
9483 return false;
9484}
9485
9486/// visitInlineAsm - Handle a call to an InlineAsm object.
9487void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9488 const BasicBlock *EHPadBB) {
9489 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9490
9491 /// ConstraintOperands - Information about all of the constraints.
9492 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9493
9497
9498 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9499 // AsmDialect, MayLoad, MayStore).
9500 bool HasSideEffect = IA->hasSideEffects();
9501 ExtraFlags ExtraInfo(Call);
9502
9503 for (auto &T : TargetConstraints) {
9504 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9505 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9506
9507 if (OpInfo.CallOperandVal)
9508 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
9509
9510 if (!HasSideEffect)
9511 HasSideEffect = OpInfo.hasMemory(TLI);
9512
9513 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9514 // FIXME: Could we compute this on OpInfo rather than T?
9515
9516 // Compute the constraint code and ConstraintType to use.
9518
9519 if (T.ConstraintType == TargetLowering::C_Immediate &&
9520 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9521 // We've delayed emitting a diagnostic like the "n" constraint because
9522 // inlining could cause an integer showing up.
9523 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
9524 "' expects an integer constant "
9525 "expression");
9526
9527 ExtraInfo.update(T);
9528 }
9529
9530 // We won't need to flush pending loads if this asm doesn't touch
9531 // memory and is nonvolatile.
9532 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9533
9534 bool EmitEHLabels = isa<InvokeInst>(Call);
9535 if (EmitEHLabels) {
9536 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9537 }
9538 bool IsCallBr = isa<CallBrInst>(Call);
9539
9540 if (IsCallBr || EmitEHLabels) {
9541 // If this is a callbr or invoke we need to flush pending exports since
9542 // inlineasm_br and invoke are terminators.
9543 // We need to do this before nodes are glued to the inlineasm_br node.
9544 Chain = getControlRoot();
9545 }
9546
9547 MCSymbol *BeginLabel = nullptr;
9548 if (EmitEHLabels) {
9549 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
9550 }
9551
9552 int OpNo = -1;
9553 SmallVector<StringRef> AsmStrs;
9554 IA->collectAsmStrs(AsmStrs);
9555
9556 // Second pass over the constraints: compute which constraint option to use.
9557 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9558 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
9559 OpNo++;
9560
9561 // If this is an output operand with a matching input operand, look up the
9562 // matching input. If their types mismatch, e.g. one is an integer, the
9563 // other is floating point, or their sizes are different, flag it as an
9564 // error.
9565 if (OpInfo.hasMatchingInput()) {
9566 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
9567 patchMatchingInput(OpInfo, Input, DAG);
9568 }
9569
9570 // Compute the constraint code and ConstraintType to use.
9571 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
9572
9573 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
9574 OpInfo.Type == InlineAsm::isClobber) ||
9575 OpInfo.ConstraintType == TargetLowering::C_Address)
9576 continue;
9577
9578 // In Linux PIC model, there are 4 cases about value/label addressing:
9579 //
9580 // 1: Function call or Label jmp inside the module.
9581 // 2: Data access (such as global variable, static variable) inside module.
9582 // 3: Function call or Label jmp outside the module.
9583 // 4: Data access (such as global variable) outside the module.
9584 //
9585 // Due to current llvm inline asm architecture designed to not "recognize"
9586 // the asm code, there are quite troubles for us to treat mem addressing
9587 // differently for same value/adress used in different instuctions.
9588 // For example, in pic model, call a func may in plt way or direclty
9589 // pc-related, but lea/mov a function adress may use got.
9590 //
9591 // Here we try to "recognize" function call for the case 1 and case 3 in
9592 // inline asm. And try to adjust the constraint for them.
9593 //
9594 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
9595 // label, so here we don't handle jmp function label now, but we need to
9596 // enhance it (especilly in PIC model) if we meet meaningful requirements.
9597 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
9598 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
9600 OpInfo.isIndirect = false;
9601 OpInfo.ConstraintType = TargetLowering::C_Address;
9602 }
9603
9604 // If this is a memory input, and if the operand is not indirect, do what we
9605 // need to provide an address for the memory input.
9606 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
9607 !OpInfo.isIndirect) {
9608 assert((OpInfo.isMultipleAlternative ||
9609 (OpInfo.Type == InlineAsm::isInput)) &&
9610 "Can only indirectify direct input operands!");
9611
9612 // Memory operands really want the address of the value.
9613 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
9614
9615 // There is no longer a Value* corresponding to this operand.
9616 OpInfo.CallOperandVal = nullptr;
9617
9618 // It is now an indirect operand.
9619 OpInfo.isIndirect = true;
9620 }
9621
9622 }
9623
9624 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
9625 std::vector<SDValue> AsmNodeOperands;
9626 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
9627 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
9628 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
9629
9630 // If we have a !srcloc metadata node associated with it, we want to attach
9631 // this to the ultimately generated inline asm machineinstr. To do this, we
9632 // pass in the third operand as this (potentially null) inline asm MDNode.
9633 const MDNode *SrcLoc = Call.getMetadata("srcloc");
9634 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
9635
9636 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
9637 // bits as operand 3.
9638 AsmNodeOperands.push_back(DAG.getTargetConstant(
9639 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9640
9641 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
9642 // this, assign virtual and physical registers for inputs and otput.
9643 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9644 // Assign Registers.
9645 SDISelAsmOperandInfo &RefOpInfo =
9646 OpInfo.isMatchingInputConstraint()
9647 ? ConstraintOperands[OpInfo.getMatchedOperand()]
9648 : OpInfo;
9649 const auto RegError =
9650 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
9651 if (RegError) {
9654 const char *RegName = TRI.getName(*RegError);
9655 emitInlineAsmError(Call, "register '" + Twine(RegName) +
9656 "' allocated for constraint '" +
9657 Twine(OpInfo.ConstraintCode) +
9658 "' does not match required type");
9659 return;
9660 }
9661
9662 auto DetectWriteToReservedRegister = [&]() {
9665 for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
9667 TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
9668 const char *RegName = TRI.getName(Reg);
9669 emitInlineAsmError(Call, "write to reserved register '" +
9670 Twine(RegName) + "'");
9671 return true;
9672 }
9673 }
9674 return false;
9675 };
9676 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
9677 (OpInfo.Type == InlineAsm::isInput &&
9678 !OpInfo.isMatchingInputConstraint())) &&
9679 "Only address as input operand is allowed.");
9680
9681 switch (OpInfo.Type) {
9683 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9684 const InlineAsm::ConstraintCode ConstraintID =
9685 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9687 "Failed to convert memory constraint code to constraint id.");
9688
9689 // Add information to the INLINEASM node to know about this output.
9691 OpFlags.setMemConstraint(ConstraintID);
9692 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
9693 MVT::i32));
9694 AsmNodeOperands.push_back(OpInfo.CallOperand);
9695 } else {
9696 // Otherwise, this outputs to a register (directly for C_Register /
9697 // C_RegisterClass, and a target-defined fashion for
9698 // C_Immediate/C_Other). Find a register that we can use.
9699 if (OpInfo.AssignedRegs.Regs.empty()) {
9700 emitInlineAsmError(
9701 Call, "couldn't allocate output register for constraint '" +
9702 Twine(OpInfo.ConstraintCode) + "'");
9703 return;
9704 }
9705
9706 if (DetectWriteToReservedRegister())
9707 return;
9708
9709 // Add information to the INLINEASM node to know that this register is
9710 // set.
9711 OpInfo.AssignedRegs.AddInlineAsmOperands(
9712 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
9714 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
9715 }
9716 break;
9717
9718 case InlineAsm::isInput:
9719 case InlineAsm::isLabel: {
9720 SDValue InOperandVal = OpInfo.CallOperand;
9721
9722 if (OpInfo.isMatchingInputConstraint()) {
9723 // If this is required to match an output register we have already set,
9724 // just use its register.
9725 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
9726 AsmNodeOperands);
9727 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
9728 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
9729 if (OpInfo.isIndirect) {
9730 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
9731 emitInlineAsmError(Call, "inline asm not supported yet: "
9732 "don't know how to handle tied "
9733 "indirect register inputs");
9734 return;
9735 }
9736
9741 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
9742 Register TiedReg = R->getReg();
9743 MVT RegVT = R->getSimpleValueType(0);
9744 const TargetRegisterClass *RC =
9745 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
9746 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
9747 : TRI.getMinimalPhysRegClass(TiedReg);
9748 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
9749 Regs.push_back(MRI.createVirtualRegister(RC));
9750
9751 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
9752
9753 SDLoc dl = getCurSDLoc();
9754 // Use the produced MatchedRegs object to
9755 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
9756 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
9757 OpInfo.getMatchedOperand(), dl, DAG,
9758 AsmNodeOperands);
9759 break;
9760 }
9761
9762 assert(Flag.isMemKind() && "Unknown matching constraint!");
9763 assert(Flag.getNumOperandRegisters() == 1 &&
9764 "Unexpected number of operands");
9765 // Add information to the INLINEASM node to know about this input.
9766 // See InlineAsm.h isUseOperandTiedToDef.
9767 Flag.clearMemConstraint();
9768 Flag.setMatchingOp(OpInfo.getMatchedOperand());
9769 AsmNodeOperands.push_back(DAG.getTargetConstant(
9770 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9771 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
9772 break;
9773 }
9774
9775 // Treat indirect 'X' constraint as memory.
9776 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
9777 OpInfo.isIndirect)
9778 OpInfo.ConstraintType = TargetLowering::C_Memory;
9779
9780 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
9781 OpInfo.ConstraintType == TargetLowering::C_Other) {
9782 std::vector<SDValue> Ops;
9783 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
9784 Ops, DAG);
9785 if (Ops.empty()) {
9786 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
9787 if (isa<ConstantSDNode>(InOperandVal)) {
9788 emitInlineAsmError(Call, "value out of range for constraint '" +
9789 Twine(OpInfo.ConstraintCode) + "'");
9790 return;
9791 }
9792
9793 emitInlineAsmError(Call,
9794 "invalid operand for inline asm constraint '" +
9795 Twine(OpInfo.ConstraintCode) + "'");
9796 return;
9797 }
9798
9799 // Add information to the INLINEASM node to know about this input.
9800 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
9801 AsmNodeOperands.push_back(DAG.getTargetConstant(
9802 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9803 llvm::append_range(AsmNodeOperands, Ops);
9804 break;
9805 }
9806
9807 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9808 assert((OpInfo.isIndirect ||
9809 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
9810 "Operand must be indirect to be a mem!");
9811 assert(InOperandVal.getValueType() ==
9813 "Memory operands expect pointer values");
9814
9815 const InlineAsm::ConstraintCode ConstraintID =
9816 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9818 "Failed to convert memory constraint code to constraint id.");
9819
9820 // Add information to the INLINEASM node to know about this input.
9822 ResOpType.setMemConstraint(ConstraintID);
9823 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
9824 getCurSDLoc(),
9825 MVT::i32));
9826 AsmNodeOperands.push_back(InOperandVal);
9827 break;
9828 }
9829
9830 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
9831 const InlineAsm::ConstraintCode ConstraintID =
9832 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9834 "Failed to convert memory constraint code to constraint id.");
9835
9837
9838 SDValue AsmOp = InOperandVal;
9839 if (isFunction(InOperandVal)) {
9840 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
9841 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
9842 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
9843 InOperandVal.getValueType(),
9844 GA->getOffset());
9845 }
9846
9847 // Add information to the INLINEASM node to know about this input.
9848 ResOpType.setMemConstraint(ConstraintID);
9849
9850 AsmNodeOperands.push_back(
9851 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
9852
9853 AsmNodeOperands.push_back(AsmOp);
9854 break;
9855 }
9856
9857 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
9858 OpInfo.ConstraintType == TargetLowering::C_Register) &&
9859 "Unknown constraint type!");
9860
9861 // TODO: Support this.
9862 if (OpInfo.isIndirect) {
9863 emitInlineAsmError(
9864 Call, "Don't know how to handle indirect register inputs yet "
9865 "for constraint '" +
9866 Twine(OpInfo.ConstraintCode) + "'");
9867 return;
9868 }
9869
9870 // Copy the input into the appropriate registers.
9871 if (OpInfo.AssignedRegs.Regs.empty()) {
9872 emitInlineAsmError(Call,
9873 "couldn't allocate input reg for constraint '" +
9874 Twine(OpInfo.ConstraintCode) + "'");
9875 return;
9876 }
9877
9878 if (DetectWriteToReservedRegister())
9879 return;
9880
9881 SDLoc dl = getCurSDLoc();
9882
9883 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
9884 &Call);
9885
9886 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
9887 0, dl, DAG, AsmNodeOperands);
9888 break;
9889 }
9891 // Add the clobbered value to the operand list, so that the register
9892 // allocator is aware that the physreg got clobbered.
9893 if (!OpInfo.AssignedRegs.Regs.empty())
9894 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,
9895 false, 0, getCurSDLoc(), DAG,
9896 AsmNodeOperands);
9897 break;
9898 }
9899 }
9900
9901 // Finish up input operands. Set the input chain and add the flag last.
9902 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9903 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
9904
9905 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9906 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9907 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9908 Glue = Chain.getValue(1);
9909
9910 // Do additional work to generate outputs.
9911
9912 SmallVector<EVT, 1> ResultVTs;
9913 SmallVector<SDValue, 1> ResultValues;
9914 SmallVector<SDValue, 8> OutChains;
9915
9916 llvm::Type *CallResultType = Call.getType();
9917 ArrayRef<Type *> ResultTypes;
9918 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9919 ResultTypes = StructResult->elements();
9920 else if (!CallResultType->isVoidTy())
9921 ResultTypes = ArrayRef(CallResultType);
9922
9923 auto CurResultType = ResultTypes.begin();
9924 auto handleRegAssign = [&](SDValue V) {
9925 assert(CurResultType != ResultTypes.end() && "Unexpected value");
9926 assert((*CurResultType)->isSized() && "Unexpected unsized type");
9927 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9928 ++CurResultType;
9929 // If the type of the inline asm call site return value is different but has
9930 // same size as the type of the asm output bitcast it. One example of this
9931 // is for vectors with different width / number of elements. This can
9932 // happen for register classes that can contain multiple different value
9933 // types. The preg or vreg allocated may not have the same VT as was
9934 // expected.
9935 //
9936 // This can also happen for a return value that disagrees with the register
9937 // class it is put in, eg. a double in a general-purpose register on a
9938 // 32-bit machine.
9939 if (ResultVT != V.getValueType() &&
9940 ResultVT.getSizeInBits() == V.getValueSizeInBits())
9941 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9942 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9943 V.getValueType().isInteger()) {
9944 // If a result value was tied to an input value, the computed result
9945 // may have a wider width than the expected result. Extract the
9946 // relevant portion.
9947 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9948 }
9949 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9950 ResultVTs.push_back(ResultVT);
9951 ResultValues.push_back(V);
9952 };
9953
9954 // Deal with output operands.
9955 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9956 if (OpInfo.Type == InlineAsm::isOutput) {
9957 SDValue Val;
9958 // Skip trivial output operands.
9959 if (OpInfo.AssignedRegs.Regs.empty())
9960 continue;
9961
9962 switch (OpInfo.ConstraintType) {
9965 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9966 Chain, &Glue, &Call);
9967 break;
9970 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
9971 OpInfo, DAG);
9972 break;
9974 break; // Already handled.
9976 break; // Silence warning.
9978 assert(false && "Unexpected unknown constraint");
9979 }
9980
9981 // Indirect output manifest as stores. Record output chains.
9982 if (OpInfo.isIndirect) {
9983 const Value *Ptr = OpInfo.CallOperandVal;
9984 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9985 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9987 OutChains.push_back(Store);
9988 } else {
9989 // generate CopyFromRegs to associated registers.
9990 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9991 if (Val.getOpcode() == ISD::MERGE_VALUES) {
9992 for (const SDValue &V : Val->op_values())
9993 handleRegAssign(V);
9994 } else
9995 handleRegAssign(Val);
9996 }
9997 }
9998 }
9999
10000 // Set results.
10001 if (!ResultValues.empty()) {
10002 assert(CurResultType == ResultTypes.end() &&
10003 "Mismatch in number of ResultTypes");
10004 assert(ResultValues.size() == ResultTypes.size() &&
10005 "Mismatch in number of output operands in asm result");
10006
10008 DAG.getVTList(ResultVTs), ResultValues);
10009 setValue(&Call, V);
10010 }
10011
10012 // Collect store chains.
10013 if (!OutChains.empty())
10014 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10015
10016 if (EmitEHLabels) {
10017 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10018 }
10019
10020 // Only Update Root if inline assembly has a memory effect.
10021 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10022 EmitEHLabels)
10023 DAG.setRoot(Chain);
10024}
10025
10026void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10027 const Twine &Message) {
10028 LLVMContext &Ctx = *DAG.getContext();
10029 Ctx.emitError(&Call, Message);
10030
10031 // Make sure we leave the DAG in a valid state
10033 SmallVector<EVT, 1> ValueVTs;
10034 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10035
10036 if (ValueVTs.empty())
10037 return;
10038
10040 for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
10041 Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
10042
10043 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10044}
10045
10046void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10048 MVT::Other, getRoot(),
10049 getValue(I.getArgOperand(0)),
10050 DAG.getSrcValue(I.getArgOperand(0))));
10051}
10052
10053void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10055 const DataLayout &DL = DAG.getDataLayout();
10057 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10058 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10059 DL.getABITypeAlign(I.getType()).value());
10060 DAG.setRoot(V.getValue(1));
10061
10062 if (I.getType()->isPointerTy())
10064 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10065 setValue(&I, V);
10066}
10067
10068void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10070 MVT::Other, getRoot(),
10071 getValue(I.getArgOperand(0)),
10072 DAG.getSrcValue(I.getArgOperand(0))));
10073}
10074
10075void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10077 MVT::Other, getRoot(),
10078 getValue(I.getArgOperand(0)),
10079 getValue(I.getArgOperand(1)),
10080 DAG.getSrcValue(I.getArgOperand(0)),
10081 DAG.getSrcValue(I.getArgOperand(1))));
10082}
10083
10085 const Instruction &I,
10086 SDValue Op) {
10087 const MDNode *Range = getRangeMetadata(I);
10088 if (!Range)
10089 return Op;
10090
10092 if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
10093 return Op;
10094
10095 APInt Lo = CR.getUnsignedMin();
10096 if (!Lo.isMinValue())
10097 return Op;
10098
10099 APInt Hi = CR.getUnsignedMax();
10100 unsigned Bits = std::max(Hi.getActiveBits(),
10101 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10102
10103 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10104
10105 SDLoc SL = getCurSDLoc();
10106
10107 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10108 DAG.getValueType(SmallVT));
10109 unsigned NumVals = Op.getNode()->getNumValues();
10110 if (NumVals == 1)
10111 return ZExt;
10112
10114
10115 Ops.push_back(ZExt);
10116 for (unsigned I = 1; I != NumVals; ++I)
10117 Ops.push_back(Op.getValue(I));
10118
10119 return DAG.getMergeValues(Ops, SL);
10120}
10121
10122/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10123/// the call being lowered.
10124///
10125/// This is a helper for lowering intrinsics that follow a target calling
10126/// convention or require stack pointer adjustment. Only a subset of the
10127/// intrinsic's operands need to participate in the calling convention.
10130 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10131 AttributeSet RetAttrs, bool IsPatchPoint) {
10133 Args.reserve(NumArgs);
10134
10135 // Populate the argument list.
10136 // Attributes for args start at offset 1, after the return attribute.
10137 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10138 ArgI != ArgE; ++ArgI) {
10139 const Value *V = Call->getOperand(ArgI);
10140
10141 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10142
10144 Entry.Node = getValue(V);
10145 Entry.Ty = V->getType();
10146 Entry.setAttributes(Call, ArgI);
10147 Args.push_back(Entry);
10148 }
10149
10151 .setChain(getRoot())
10152 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10153 RetAttrs)
10154 .setDiscardResult(Call->use_empty())
10155 .setIsPatchPoint(IsPatchPoint)
10157 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10158}
10159
10160/// Add a stack map intrinsic call's live variable operands to a stackmap
10161/// or patchpoint target node's operand list.
10162///
10163/// Constants are converted to TargetConstants purely as an optimization to
10164/// avoid constant materialization and register allocation.
10165///
10166/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10167/// generate addess computation nodes, and so FinalizeISel can convert the
10168/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10169/// address materialization and register allocation, but may also be required
10170/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10171/// alloca in the entry block, then the runtime may assume that the alloca's
10172/// StackMap location can be read immediately after compilation and that the
10173/// location is valid at any point during execution (this is similar to the
10174/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10175/// only available in a register, then the runtime would need to trap when
10176/// execution reaches the StackMap in order to read the alloca's location.
10177static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10178 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10179 SelectionDAGBuilder &Builder) {
10180 SelectionDAG &DAG = Builder.DAG;
10181 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10182 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10183
10184 // Things on the stack are pointer-typed, meaning that they are already
10185 // legal and can be emitted directly to target nodes.
10186 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
10187 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10188 } else {
10189 // Otherwise emit a target independent node to be legalised.
10190 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10191 }
10192 }
10193}
10194
10195/// Lower llvm.experimental.stackmap.
10196void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10197 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10198 // [live variables...])
10199
10200 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10201
10202 SDValue Chain, InGlue, Callee;
10204
10205 SDLoc DL = getCurSDLoc();
10207
10208 // The stackmap intrinsic only records the live variables (the arguments
10209 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10210 // intrinsic, this won't be lowered to a function call. This means we don't
10211 // have to worry about calling conventions and target specific lowering code.
10212 // Instead we perform the call lowering right here.
10213 //
10214 // chain, flag = CALLSEQ_START(chain, 0, 0)
10215 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10216 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10217 //
10218 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10219 InGlue = Chain.getValue(1);
10220
10221 // Add the STACKMAP operands, starting with DAG house-keeping.
10222 Ops.push_back(Chain);
10223 Ops.push_back(InGlue);
10224
10225 // Add the <id>, <numShadowBytes> operands.
10226 //
10227 // These do not require legalisation, and can be emitted directly to target
10228 // constant nodes.
10230 assert(ID.getValueType() == MVT::i64);
10231 SDValue IDConst =
10232 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10233 Ops.push_back(IDConst);
10234
10235 SDValue Shad = getValue(CI.getArgOperand(1));
10236 assert(Shad.getValueType() == MVT::i32);
10237 SDValue ShadConst =
10239 Ops.push_back(ShadConst);
10240
10241 // Add the live variables.
10242 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10243
10244 // Create the STACKMAP node.
10245 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10246 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10247 InGlue = Chain.getValue(1);
10248
10249 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10250
10251 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10252
10253 // Set the root to the target-lowered call chain.
10254 DAG.setRoot(Chain);
10255
10256 // Inform the Frame Information that we have a stackmap in this function.
10258}
10259
10260/// Lower llvm.experimental.patchpoint directly to its target opcode.
10261void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10262 const BasicBlock *EHPadBB) {
10263 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10264 // i32 <numBytes>,
10265 // i8* <target>,
10266 // i32 <numArgs>,
10267 // [Args...],
10268 // [live variables...])
10269
10271 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10272 bool HasDef = !CB.getType()->isVoidTy();
10273 SDLoc dl = getCurSDLoc();
10275
10276 // Handle immediate and symbolic callees.
10277 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10278 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10279 /*isTarget=*/true);
10280 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10281 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10282 SDLoc(SymbolicCallee),
10283 SymbolicCallee->getValueType(0));
10284
10285 // Get the real number of arguments participating in the call <numArgs>
10287 unsigned NumArgs = NArgVal->getAsZExtVal();
10288
10289 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10290 // Intrinsics include all meta-operands up to but not including CC.
10291 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10292 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10293 "Not enough arguments provided to the patchpoint intrinsic");
10294
10295 // For AnyRegCC the arguments are lowered later on manually.
10296 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10297 Type *ReturnTy =
10298 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10299
10301 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10302 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10303 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10304
10305 SDNode *CallEnd = Result.second.getNode();
10306 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10307 CallEnd = CallEnd->getOperand(0).getNode();
10308
10309 /// Get a call instruction from the call sequence chain.
10310 /// Tail calls are not allowed.
10311 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10312 "Expected a callseq node.");
10313 SDNode *Call = CallEnd->getOperand(0).getNode();
10314 bool HasGlue = Call->getGluedNode();
10315
10316 // Replace the target specific call node with the patchable intrinsic.
10318
10319 // Push the chain.
10320 Ops.push_back(*(Call->op_begin()));
10321
10322 // Optionally, push the glue (if any).
10323 if (HasGlue)
10324 Ops.push_back(*(Call->op_end() - 1));
10325
10326 // Push the register mask info.
10327 if (HasGlue)
10328 Ops.push_back(*(Call->op_end() - 2));
10329 else
10330 Ops.push_back(*(Call->op_end() - 1));
10331
10332 // Add the <id> and <numBytes> constants.
10334 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10336 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10337
10338 // Add the callee.
10339 Ops.push_back(Callee);
10340
10341 // Adjust <numArgs> to account for any arguments that have been passed on the
10342 // stack instead.
10343 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10344 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10345 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10346 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10347
10348 // Add the calling convention
10349 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10350
10351 // Add the arguments we omitted previously. The register allocator should
10352 // place these in any free register.
10353 if (IsAnyRegCC)
10354 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10355 Ops.push_back(getValue(CB.getArgOperand(i)));
10356
10357 // Push the arguments from the call instruction.
10358 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10359 Ops.append(Call->op_begin() + 2, e);
10360
10361 // Push live variables for the stack map.
10362 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10363
10364 SDVTList NodeTys;
10365 if (IsAnyRegCC && HasDef) {
10366 // Create the return types based on the intrinsic definition
10368 SmallVector<EVT, 3> ValueVTs;
10369 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10370 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10371
10372 // There is always a chain and a glue type at the end
10373 ValueVTs.push_back(MVT::Other);
10374 ValueVTs.push_back(MVT::Glue);
10375 NodeTys = DAG.getVTList(ValueVTs);
10376 } else
10377 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10378
10379 // Replace the target specific call node with a PATCHPOINT node.
10380 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10381
10382 // Update the NodeMap.
10383 if (HasDef) {
10384 if (IsAnyRegCC)
10385 setValue(&CB, SDValue(PPV.getNode(), 0));
10386 else
10387 setValue(&CB, Result.first);
10388 }
10389
10390 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10391 // call sequence. Furthermore the location of the chain and glue can change
10392 // when the AnyReg calling convention is used and the intrinsic returns a
10393 // value.
10394 if (IsAnyRegCC && HasDef) {
10395 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10396 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10398 } else
10399 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10400 DAG.DeleteNode(Call);
10401
10402 // Inform the Frame Information that we have a patchpoint in this function.
10404}
10405
10406void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10407 unsigned Intrinsic) {
10409 SDValue Op1 = getValue(I.getArgOperand(0));
10410 SDValue Op2;
10411 if (I.arg_size() > 1)
10412 Op2 = getValue(I.getArgOperand(1));
10413 SDLoc dl = getCurSDLoc();
10414 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10415 SDValue Res;
10416 SDNodeFlags SDFlags;
10417 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10418 SDFlags.copyFMF(*FPMO);
10419
10420 switch (Intrinsic) {
10421 case Intrinsic::vector_reduce_fadd:
10422 if (SDFlags.hasAllowReassociation())
10423 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10424 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10425 SDFlags);
10426 else
10427 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10428 break;
10429 case Intrinsic::vector_reduce_fmul:
10430 if (SDFlags.hasAllowReassociation())
10431 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10432 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10433 SDFlags);
10434 else
10435 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10436 break;
10437 case Intrinsic::vector_reduce_add:
10438 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10439 break;
10440 case Intrinsic::vector_reduce_mul:
10441 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10442 break;
10443 case Intrinsic::vector_reduce_and:
10444 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10445 break;
10446 case Intrinsic::vector_reduce_or:
10447 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10448 break;
10449 case Intrinsic::vector_reduce_xor:
10450 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10451 break;
10452 case Intrinsic::vector_reduce_smax:
10453 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10454 break;
10455 case Intrinsic::vector_reduce_smin:
10456 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10457 break;
10458 case Intrinsic::vector_reduce_umax:
10459 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10460 break;
10461 case Intrinsic::vector_reduce_umin:
10462 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10463 break;
10464 case Intrinsic::vector_reduce_fmax:
10465 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10466 break;
10467 case Intrinsic::vector_reduce_fmin:
10468 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10469 break;
10470 case Intrinsic::vector_reduce_fmaximum:
10471 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10472 break;
10473 case Intrinsic::vector_reduce_fminimum:
10474 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10475 break;
10476 default:
10477 llvm_unreachable("Unhandled vector reduce intrinsic");
10478 }
10479 setValue(&I, Res);
10480}
10481
10482/// Returns an AttributeList representing the attributes applied to the return
10483/// value of the given call.
10486 if (CLI.RetSExt)
10487 Attrs.push_back(Attribute::SExt);
10488 if (CLI.RetZExt)
10489 Attrs.push_back(Attribute::ZExt);
10490 if (CLI.IsInReg)
10491 Attrs.push_back(Attribute::InReg);
10492
10494 Attrs);
10495}
10496
10497/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10498/// implementation, which just calls LowerCall.
10499/// FIXME: When all targets are
10500/// migrated to using LowerCall, this hook should be integrated into SDISel.
10501std::pair<SDValue, SDValue>
10503 // Handle the incoming return values from the call.
10504 CLI.Ins.clear();
10505 Type *OrigRetTy = CLI.RetTy;
10506 SmallVector<EVT, 4> RetTys;
10508 auto &DL = CLI.DAG.getDataLayout();
10509 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
10510
10511 if (CLI.IsPostTypeLegalization) {
10512 // If we are lowering a libcall after legalization, split the return type.
10513 SmallVector<EVT, 4> OldRetTys;
10514 SmallVector<TypeSize, 4> OldOffsets;
10515 RetTys.swap(OldRetTys);
10516 Offsets.swap(OldOffsets);
10517
10518 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10519 EVT RetVT = OldRetTys[i];
10520 uint64_t Offset = OldOffsets[i];
10521 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
10522 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
10523 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10524 RetTys.append(NumRegs, RegisterVT);
10525 for (unsigned j = 0; j != NumRegs; ++j)
10526 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
10527 }
10528 }
10529
10531 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
10532
10533 bool CanLowerReturn =
10535 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
10536
10537 SDValue DemoteStackSlot;
10538 int DemoteStackIdx = -100;
10539 if (!CanLowerReturn) {
10540 // FIXME: equivalent assert?
10541 // assert(!CS.hasInAllocaArgument() &&
10542 // "sret demotion is incompatible with inalloca");
10543 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
10544 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
10546 DemoteStackIdx =
10547 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
10548 Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
10549 DL.getAllocaAddrSpace());
10550
10551 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
10552 ArgListEntry Entry;
10553 Entry.Node = DemoteStackSlot;
10554 Entry.Ty = StackSlotPtrType;
10555 Entry.IsSExt = false;
10556 Entry.IsZExt = false;
10557 Entry.IsInReg = false;
10558 Entry.IsSRet = true;
10559 Entry.IsNest = false;
10560 Entry.IsByVal = false;
10561 Entry.IsByRef = false;
10562 Entry.IsReturned = false;
10563 Entry.IsSwiftSelf = false;
10564 Entry.IsSwiftAsync = false;
10565 Entry.IsSwiftError = false;
10566 Entry.IsCFGuardTarget = false;
10567 Entry.Alignment = Alignment;
10568 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
10569 CLI.NumFixedArgs += 1;
10570 CLI.getArgs()[0].IndirectType = CLI.RetTy;
10571 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
10572
10573 // sret demotion isn't compatible with tail-calls, since the sret argument
10574 // points into the callers stack frame.
10575 CLI.IsTailCall = false;
10576 } else {
10577 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10578 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
10579 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10580 ISD::ArgFlagsTy Flags;
10581 if (NeedsRegBlock) {
10582 Flags.setInConsecutiveRegs();
10583 if (I == RetTys.size() - 1)
10584 Flags.setInConsecutiveRegsLast();
10585 }
10586 EVT VT = RetTys[I];
10588 CLI.CallConv, VT);
10589 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10590 CLI.CallConv, VT);
10591 for (unsigned i = 0; i != NumRegs; ++i) {
10592 ISD::InputArg MyFlags;
10593 MyFlags.Flags = Flags;
10594 MyFlags.VT = RegisterVT;
10595 MyFlags.ArgVT = VT;
10596 MyFlags.Used = CLI.IsReturnValueUsed;
10597 if (CLI.RetTy->isPointerTy()) {
10598 MyFlags.Flags.setPointer();
10599 MyFlags.Flags.setPointerAddrSpace(
10600 cast<PointerType>(CLI.RetTy)->getAddressSpace());
10601 }
10602 if (CLI.RetSExt)
10603 MyFlags.Flags.setSExt();
10604 if (CLI.RetZExt)
10605 MyFlags.Flags.setZExt();
10606 if (CLI.IsInReg)
10607 MyFlags.Flags.setInReg();
10608 CLI.Ins.push_back(MyFlags);
10609 }
10610 }
10611 }
10612
10613 // We push in swifterror return as the last element of CLI.Ins.
10614 ArgListTy &Args = CLI.getArgs();
10615 if (supportSwiftError()) {
10616 for (const ArgListEntry &Arg : Args) {
10617 if (Arg.IsSwiftError) {
10618 ISD::InputArg MyFlags;
10619 MyFlags.VT = getPointerTy(DL);
10620 MyFlags.ArgVT = EVT(getPointerTy(DL));
10621 MyFlags.Flags.setSwiftError();
10622 CLI.Ins.push_back(MyFlags);
10623 }
10624 }
10625 }
10626
10627 // Handle all of the outgoing arguments.
10628 CLI.Outs.clear();
10629 CLI.OutVals.clear();
10630 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
10631 SmallVector<EVT, 4> ValueVTs;
10632 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
10633 // FIXME: Split arguments if CLI.IsPostTypeLegalization
10634 Type *FinalType = Args[i].Ty;
10635 if (Args[i].IsByVal)
10636 FinalType = Args[i].IndirectType;
10637 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10638 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
10639 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
10640 ++Value) {
10641 EVT VT = ValueVTs[Value];
10642 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
10643 SDValue Op = SDValue(Args[i].Node.getNode(),
10644 Args[i].Node.getResNo() + Value);
10645 ISD::ArgFlagsTy Flags;
10646
10647 // Certain targets (such as MIPS), may have a different ABI alignment
10648 // for a type depending on the context. Give the target a chance to
10649 // specify the alignment it wants.
10650 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
10651 Flags.setOrigAlign(OriginalAlignment);
10652
10653 if (Args[i].Ty->isPointerTy()) {
10654 Flags.setPointer();
10655 Flags.setPointerAddrSpace(
10656 cast<PointerType>(Args[i].Ty)->getAddressSpace());
10657 }
10658 if (Args[i].IsZExt)
10659 Flags.setZExt();
10660 if (Args[i].IsSExt)
10661 Flags.setSExt();
10662 if (Args[i].IsInReg) {
10663 // If we are using vectorcall calling convention, a structure that is
10664 // passed InReg - is surely an HVA
10666 isa<StructType>(FinalType)) {
10667 // The first value of a structure is marked
10668 if (0 == Value)
10669 Flags.setHvaStart();
10670 Flags.setHva();
10671 }
10672 // Set InReg Flag
10673 Flags.setInReg();
10674 }
10675 if (Args[i].IsSRet)
10676 Flags.setSRet();
10677 if (Args[i].IsSwiftSelf)
10678 Flags.setSwiftSelf();
10679 if (Args[i].IsSwiftAsync)
10680 Flags.setSwiftAsync();
10681 if (Args[i].IsSwiftError)
10682 Flags.setSwiftError();
10683 if (Args[i].IsCFGuardTarget)
10684 Flags.setCFGuardTarget();
10685 if (Args[i].IsByVal)
10686 Flags.setByVal();
10687 if (Args[i].IsByRef)
10688 Flags.setByRef();
10689 if (Args[i].IsPreallocated) {
10690 Flags.setPreallocated();
10691 // Set the byval flag for CCAssignFn callbacks that don't know about
10692 // preallocated. This way we can know how many bytes we should've
10693 // allocated and how many bytes a callee cleanup function will pop. If
10694 // we port preallocated to more targets, we'll have to add custom
10695 // preallocated handling in the various CC lowering callbacks.
10696 Flags.setByVal();
10697 }
10698 if (Args[i].IsInAlloca) {
10699 Flags.setInAlloca();
10700 // Set the byval flag for CCAssignFn callbacks that don't know about
10701 // inalloca. This way we can know how many bytes we should've allocated
10702 // and how many bytes a callee cleanup function will pop. If we port
10703 // inalloca to more targets, we'll have to add custom inalloca handling
10704 // in the various CC lowering callbacks.
10705 Flags.setByVal();
10706 }
10707 Align MemAlign;
10708 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
10709 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
10710 Flags.setByValSize(FrameSize);
10711
10712 // info is not there but there are cases it cannot get right.
10713 if (auto MA = Args[i].Alignment)
10714 MemAlign = *MA;
10715 else
10716 MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
10717 } else if (auto MA = Args[i].Alignment) {
10718 MemAlign = *MA;
10719 } else {
10720 MemAlign = OriginalAlignment;
10721 }
10722 Flags.setMemAlign(MemAlign);
10723 if (Args[i].IsNest)
10724 Flags.setNest();
10725 if (NeedsRegBlock)
10726 Flags.setInConsecutiveRegs();
10727
10729 CLI.CallConv, VT);
10730 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10731 CLI.CallConv, VT);
10732 SmallVector<SDValue, 4> Parts(NumParts);
10733 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
10734
10735 if (Args[i].IsSExt)
10736 ExtendKind = ISD::SIGN_EXTEND;
10737 else if (Args[i].IsZExt)
10738 ExtendKind = ISD::ZERO_EXTEND;
10739
10740 // Conservatively only handle 'returned' on non-vectors that can be lowered,
10741 // for now.
10742 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
10744 assert((CLI.RetTy == Args[i].Ty ||
10745 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
10747 Args[i].Ty->getPointerAddressSpace())) &&
10748 RetTys.size() == NumValues && "unexpected use of 'returned'");
10749 // Before passing 'returned' to the target lowering code, ensure that
10750 // either the register MVT and the actual EVT are the same size or that
10751 // the return value and argument are extended in the same way; in these
10752 // cases it's safe to pass the argument register value unchanged as the
10753 // return register value (although it's at the target's option whether
10754 // to do so)
10755 // TODO: allow code generation to take advantage of partially preserved
10756 // registers rather than clobbering the entire register when the
10757 // parameter extension method is not compatible with the return
10758 // extension method
10759 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
10760 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
10761 CLI.RetZExt == Args[i].IsZExt))
10762 Flags.setReturned();
10763 }
10764
10765 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
10766 CLI.CallConv, ExtendKind);
10767
10768 for (unsigned j = 0; j != NumParts; ++j) {
10769 // if it isn't first piece, alignment must be 1
10770 // For scalable vectors the scalable part is currently handled
10771 // by individual targets, so we just use the known minimum size here.
10772 ISD::OutputArg MyFlags(
10773 Flags, Parts[j].getValueType().getSimpleVT(), VT,
10774 i < CLI.NumFixedArgs, i,
10775 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
10776 if (NumParts > 1 && j == 0)
10777 MyFlags.Flags.setSplit();
10778 else if (j != 0) {
10779 MyFlags.Flags.setOrigAlign(Align(1));
10780 if (j == NumParts - 1)
10781 MyFlags.Flags.setSplitEnd();
10782 }
10783
10784 CLI.Outs.push_back(MyFlags);
10785 CLI.OutVals.push_back(Parts[j]);
10786 }
10787
10788 if (NeedsRegBlock && Value == NumValues - 1)
10789 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
10790 }
10791 }
10792
10794 CLI.Chain = LowerCall(CLI, InVals);
10795
10796 // Update CLI.InVals to use outside of this function.
10797 CLI.InVals = InVals;
10798
10799 // Verify that the target's LowerCall behaved as expected.
10800 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
10801 "LowerCall didn't return a valid chain!");
10802 assert((!CLI.IsTailCall || InVals.empty()) &&
10803 "LowerCall emitted a return value for a tail call!");
10804 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
10805 "LowerCall didn't emit the correct number of values!");
10806
10807 // For a tail call, the return value is merely live-out and there aren't
10808 // any nodes in the DAG representing it. Return a special value to
10809 // indicate that a tail call has been emitted and no more Instructions
10810 // should be processed in the current block.
10811 if (CLI.IsTailCall) {
10812 CLI.DAG.setRoot(CLI.Chain);
10813 return std::make_pair(SDValue(), SDValue());
10814 }
10815
10816#ifndef NDEBUG
10817 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
10818 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
10819 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
10820 "LowerCall emitted a value with the wrong type!");
10821 }
10822#endif
10823
10824 SmallVector<SDValue, 4> ReturnValues;
10825 if (!CanLowerReturn) {
10826 // The instruction result is the result of loading from the
10827 // hidden sret parameter.
10829 Type *PtrRetTy =
10830 PointerType::get(OrigRetTy->getContext(), DL.getAllocaAddrSpace());
10831
10832 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
10833 assert(PVTs.size() == 1 && "Pointers should fit in one register");
10834 EVT PtrVT = PVTs[0];
10835
10836 unsigned NumValues = RetTys.size();
10837 ReturnValues.resize(NumValues);
10838 SmallVector<SDValue, 4> Chains(NumValues);
10839
10840 // An aggregate return value cannot wrap around the address space, so
10841 // offsets to its parts don't wrap either.
10842 SDNodeFlags Flags;
10843 Flags.setNoUnsignedWrap(true);
10844
10846 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
10847 for (unsigned i = 0; i < NumValues; ++i) {
10848 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
10849 CLI.DAG.getConstant(Offsets[i], CLI.DL,
10850 PtrVT), Flags);
10851 SDValue L = CLI.DAG.getLoad(
10852 RetTys[i], CLI.DL, CLI.Chain, Add,
10854 DemoteStackIdx, Offsets[i]),
10855 HiddenSRetAlign);
10856 ReturnValues[i] = L;
10857 Chains[i] = L.getValue(1);
10858 }
10859
10860 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
10861 } else {
10862 // Collect the legal value parts into potentially illegal values
10863 // that correspond to the original function's return values.
10864 std::optional<ISD::NodeType> AssertOp;
10865 if (CLI.RetSExt)
10866 AssertOp = ISD::AssertSext;
10867 else if (CLI.RetZExt)
10868 AssertOp = ISD::AssertZext;
10869 unsigned CurReg = 0;
10870 for (EVT VT : RetTys) {
10872 CLI.CallConv, VT);
10873 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10874 CLI.CallConv, VT);
10875
10876 ReturnValues.push_back(getCopyFromParts(
10877 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
10878 CLI.Chain, CLI.CallConv, AssertOp));
10879 CurReg += NumRegs;
10880 }
10881
10882 // For a function returning void, there is no return value. We can't create
10883 // such a node, so we just return a null return value in that case. In
10884 // that case, nothing will actually look at the value.
10885 if (ReturnValues.empty())
10886 return std::make_pair(SDValue(), CLI.Chain);
10887 }
10888
10889 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10890 CLI.DAG.getVTList(RetTys), ReturnValues);
10891 return std::make_pair(Res, CLI.Chain);
10892}
10893
10894/// Places new result values for the node in Results (their number
10895/// and types must exactly match those of the original return values of
10896/// the node), or leaves Results empty, which indicates that the node is not
10897/// to be custom lowered after all.
10900 SelectionDAG &DAG) const {
10901 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10902
10903 if (!Res.getNode())
10904 return;
10905
10906 // If the original node has one result, take the return value from
10907 // LowerOperation as is. It might not be result number 0.
10908 if (N->getNumValues() == 1) {
10909 Results.push_back(Res);
10910 return;
10911 }
10912
10913 // If the original node has multiple results, then the return node should
10914 // have the same number of results.
10915 assert((N->getNumValues() == Res->getNumValues()) &&
10916 "Lowering returned the wrong number of results!");
10917
10918 // Places new result values base on N result number.
10919 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10920 Results.push_back(Res.getValue(I));
10921}
10922
10924 llvm_unreachable("LowerOperation not implemented for this target!");
10925}
10926
10928 unsigned Reg,
10929 ISD::NodeType ExtendType) {
10931 assert((Op.getOpcode() != ISD::CopyFromReg ||
10932 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10933 "Copy from a reg to the same reg!");
10934 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10935
10937 // If this is an InlineAsm we have to match the registers required, not the
10938 // notional registers required by the type.
10939
10940 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10941 std::nullopt); // This is not an ABI copy.
10942 SDValue Chain = DAG.getEntryNode();
10943
10944 if (ExtendType == ISD::ANY_EXTEND) {
10945 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10946 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10947 ExtendType = PreferredExtendIt->second;
10948 }
10949 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10950 PendingExports.push_back(Chain);
10951}
10952
10954
10955/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10956/// entry block, return true. This includes arguments used by switches, since
10957/// the switch may expand into multiple basic blocks.
10958static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10959 // With FastISel active, we may be splitting blocks, so force creation
10960 // of virtual registers for all non-dead arguments.
10961 if (FastISel)
10962 return A->use_empty();
10963
10964 const BasicBlock &Entry = A->getParent()->front();
10965 for (const User *U : A->users())
10966 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10967 return false; // Use not in entry block.
10968
10969 return true;
10970}
10971
10973 DenseMap<const Argument *,
10974 std::pair<const AllocaInst *, const StoreInst *>>;
10975
10976/// Scan the entry block of the function in FuncInfo for arguments that look
10977/// like copies into a local alloca. Record any copied arguments in
10978/// ArgCopyElisionCandidates.
10979static void
10981 FunctionLoweringInfo *FuncInfo,
10982 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10983 // Record the state of every static alloca used in the entry block. Argument
10984 // allocas are all used in the entry block, so we need approximately as many
10985 // entries as we have arguments.
10986 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10988 unsigned NumArgs = FuncInfo->Fn->arg_size();
10989 StaticAllocas.reserve(NumArgs * 2);
10990
10991 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10992 if (!V)
10993 return nullptr;
10994 V = V->stripPointerCasts();
10995 const auto *AI = dyn_cast<AllocaInst>(V);
10996 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10997 return nullptr;
10998 auto Iter = StaticAllocas.insert({AI, Unknown});
10999 return &Iter.first->second;
11000 };
11001
11002 // Look for stores of arguments to static allocas. Look through bitcasts and
11003 // GEPs to handle type coercions, as long as the alloca is fully initialized
11004 // by the store. Any non-store use of an alloca escapes it and any subsequent
11005 // unanalyzed store might write it.
11006 // FIXME: Handle structs initialized with multiple stores.
11007 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11008 // Look for stores, and handle non-store uses conservatively.
11009 const auto *SI = dyn_cast<StoreInst>(&I);
11010 if (!SI) {
11011 // We will look through cast uses, so ignore them completely.
11012 if (I.isCast())
11013 continue;
11014 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11015 // to allocas.
11016 if (I.isDebugOrPseudoInst())
11017 continue;
11018 // This is an unknown instruction. Assume it escapes or writes to all
11019 // static alloca operands.
11020 for (const Use &U : I.operands()) {
11021 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11022 *Info = StaticAllocaInfo::Clobbered;
11023 }
11024 continue;
11025 }
11026
11027 // If the stored value is a static alloca, mark it as escaped.
11028 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11029 *Info = StaticAllocaInfo::Clobbered;
11030
11031 // Check if the destination is a static alloca.
11032 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11033 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11034 if (!Info)
11035 continue;
11036 const AllocaInst *AI = cast<AllocaInst>(Dst);
11037
11038 // Skip allocas that have been initialized or clobbered.
11039 if (*Info != StaticAllocaInfo::Unknown)
11040 continue;
11041
11042 // Check if the stored value is an argument, and that this store fully
11043 // initializes the alloca.
11044 // If the argument type has padding bits we can't directly forward a pointer
11045 // as the upper bits may contain garbage.
11046 // Don't elide copies from the same argument twice.
11047 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11048 const auto *Arg = dyn_cast<Argument>(Val);
11049 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11050 Arg->getType()->isEmptyTy() ||
11051 DL.getTypeStoreSize(Arg->getType()) !=
11052 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11053 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11054 ArgCopyElisionCandidates.count(Arg)) {
11055 *Info = StaticAllocaInfo::Clobbered;
11056 continue;
11057 }
11058
11059 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11060 << '\n');
11061
11062 // Mark this alloca and store for argument copy elision.
11063 *Info = StaticAllocaInfo::Elidable;
11064 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11065
11066 // Stop scanning if we've seen all arguments. This will happen early in -O0
11067 // builds, which is useful, because -O0 builds have large entry blocks and
11068 // many allocas.
11069 if (ArgCopyElisionCandidates.size() == NumArgs)
11070 break;
11071 }
11072}
11073
11074/// Try to elide argument copies from memory into a local alloca. Succeeds if
11075/// ArgVal is a load from a suitable fixed stack object.
11078 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11079 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11080 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11081 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11082 // Check if this is a load from a fixed stack object.
11083 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11084 if (!LNode)
11085 return;
11086 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11087 if (!FINode)
11088 return;
11089
11090 // Check that the fixed stack object is the right size and alignment.
11091 // Look at the alignment that the user wrote on the alloca instead of looking
11092 // at the stack object.
11093 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11094 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11095 const AllocaInst *AI = ArgCopyIter->second.first;
11096 int FixedIndex = FINode->getIndex();
11097 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11098 int OldIndex = AllocaIndex;
11099 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11100 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11101 LLVM_DEBUG(
11102 dbgs() << " argument copy elision failed due to bad fixed stack "
11103 "object size\n");
11104 return;
11105 }
11106 Align RequiredAlignment = AI->getAlign();
11107 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11108 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11109 "greater than stack argument alignment ("
11110 << DebugStr(RequiredAlignment) << " vs "
11111 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11112 return;
11113 }
11114
11115 // Perform the elision. Delete the old stack object and replace its only use
11116 // in the variable info map. Mark the stack object as mutable.
11117 LLVM_DEBUG({
11118 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11119 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11120 << '\n';
11121 });
11122 MFI.RemoveStackObject(OldIndex);
11123 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11124 AllocaIndex = FixedIndex;
11125 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11126 for (SDValue ArgVal : ArgVals)
11127 Chains.push_back(ArgVal.getValue(1));
11128
11129 // Avoid emitting code for the store implementing the copy.
11130 const StoreInst *SI = ArgCopyIter->second.second;
11131 ElidedArgCopyInstrs.insert(SI);
11132
11133 // Check for uses of the argument again so that we can avoid exporting ArgVal
11134 // if it is't used by anything other than the store.
11135 for (const Value *U : Arg.users()) {
11136 if (U != SI) {
11137 ArgHasUses = true;
11138 break;
11139 }
11140 }
11141}
11142
11143void SelectionDAGISel::LowerArguments(const Function &F) {
11144 SelectionDAG &DAG = SDB->DAG;
11145 SDLoc dl = SDB->getCurSDLoc();
11146 const DataLayout &DL = DAG.getDataLayout();
11148
11149 // In Naked functions we aren't going to save any registers.
11150 if (F.hasFnAttribute(Attribute::Naked))
11151 return;
11152
11153 if (!FuncInfo->CanLowerReturn) {
11154 // Put in an sret pointer parameter before all the other parameters.
11155 SmallVector<EVT, 1> ValueVTs;
11157 PointerType::get(F.getContext(),
11159 ValueVTs);
11160
11161 // NOTE: Assuming that a pointer will never break down to more than one VT
11162 // or one register.
11164 Flags.setSRet();
11165 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
11166 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
11168 Ins.push_back(RetArg);
11169 }
11170
11171 // Look for stores of arguments to static allocas. Mark such arguments with a
11172 // flag to ask the target to give us the memory location of that argument if
11173 // available.
11174 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11176 ArgCopyElisionCandidates);
11177
11178 // Set up the incoming argument description vector.
11179 for (const Argument &Arg : F.args()) {
11180 unsigned ArgNo = Arg.getArgNo();
11181 SmallVector<EVT, 4> ValueVTs;
11182 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11183 bool isArgValueUsed = !Arg.use_empty();
11184 unsigned PartBase = 0;
11185 Type *FinalType = Arg.getType();
11186 if (Arg.hasAttribute(Attribute::ByVal))
11187 FinalType = Arg.getParamByValType();
11188 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11189 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11190 for (unsigned Value = 0, NumValues = ValueVTs.size();
11191 Value != NumValues; ++Value) {
11192 EVT VT = ValueVTs[Value];
11193 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11195
11196
11197 if (Arg.getType()->isPointerTy()) {
11198 Flags.setPointer();
11199 Flags.setPointerAddrSpace(
11200 cast<PointerType>(Arg.getType())->getAddressSpace());
11201 }
11202 if (Arg.hasAttribute(Attribute::ZExt))
11203 Flags.setZExt();
11204 if (Arg.hasAttribute(Attribute::SExt))
11205 Flags.setSExt();
11206 if (Arg.hasAttribute(Attribute::InReg)) {
11207 // If we are using vectorcall calling convention, a structure that is
11208 // passed InReg - is surely an HVA
11209 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11210 isa<StructType>(Arg.getType())) {
11211 // The first value of a structure is marked
11212 if (0 == Value)
11213 Flags.setHvaStart();
11214 Flags.setHva();
11215 }
11216 // Set InReg Flag
11217 Flags.setInReg();
11218 }
11219 if (Arg.hasAttribute(Attribute::StructRet))
11220 Flags.setSRet();
11221 if (Arg.hasAttribute(Attribute::SwiftSelf))
11222 Flags.setSwiftSelf();
11223 if (Arg.hasAttribute(Attribute::SwiftAsync))
11224 Flags.setSwiftAsync();
11225 if (Arg.hasAttribute(Attribute::SwiftError))
11226 Flags.setSwiftError();
11227 if (Arg.hasAttribute(Attribute::ByVal))
11228 Flags.setByVal();
11229 if (Arg.hasAttribute(Attribute::ByRef))
11230 Flags.setByRef();
11231 if (Arg.hasAttribute(Attribute::InAlloca)) {
11232 Flags.setInAlloca();
11233 // Set the byval flag for CCAssignFn callbacks that don't know about
11234 // inalloca. This way we can know how many bytes we should've allocated
11235 // and how many bytes a callee cleanup function will pop. If we port
11236 // inalloca to more targets, we'll have to add custom inalloca handling
11237 // in the various CC lowering callbacks.
11238 Flags.setByVal();
11239 }
11240 if (Arg.hasAttribute(Attribute::Preallocated)) {
11241 Flags.setPreallocated();
11242 // Set the byval flag for CCAssignFn callbacks that don't know about
11243 // preallocated. This way we can know how many bytes we should've
11244 // allocated and how many bytes a callee cleanup function will pop. If
11245 // we port preallocated to more targets, we'll have to add custom
11246 // preallocated handling in the various CC lowering callbacks.
11247 Flags.setByVal();
11248 }
11249
11250 // Certain targets (such as MIPS), may have a different ABI alignment
11251 // for a type depending on the context. Give the target a chance to
11252 // specify the alignment it wants.
11253 const Align OriginalAlignment(
11255 Flags.setOrigAlign(OriginalAlignment);
11256
11257 Align MemAlign;
11258 Type *ArgMemTy = nullptr;
11259 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11260 Flags.isByRef()) {
11261 if (!ArgMemTy)
11262 ArgMemTy = Arg.getPointeeInMemoryValueType();
11263
11264 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11265
11266 // For in-memory arguments, size and alignment should be passed from FE.
11267 // BE will guess if this info is not there but there are cases it cannot
11268 // get right.
11269 if (auto ParamAlign = Arg.getParamStackAlign())
11270 MemAlign = *ParamAlign;
11271 else if ((ParamAlign = Arg.getParamAlign()))
11272 MemAlign = *ParamAlign;
11273 else
11274 MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
11275 if (Flags.isByRef())
11276 Flags.setByRefSize(MemSize);
11277 else
11278 Flags.setByValSize(MemSize);
11279 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11280 MemAlign = *ParamAlign;
11281 } else {
11282 MemAlign = OriginalAlignment;
11283 }
11284 Flags.setMemAlign(MemAlign);
11285
11286 if (Arg.hasAttribute(Attribute::Nest))
11287 Flags.setNest();
11288 if (NeedsRegBlock)
11289 Flags.setInConsecutiveRegs();
11290 if (ArgCopyElisionCandidates.count(&Arg))
11291 Flags.setCopyElisionCandidate();
11292 if (Arg.hasAttribute(Attribute::Returned))
11293 Flags.setReturned();
11294
11296 *CurDAG->getContext(), F.getCallingConv(), VT);
11297 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11298 *CurDAG->getContext(), F.getCallingConv(), VT);
11299 for (unsigned i = 0; i != NumRegs; ++i) {
11300 // For scalable vectors, use the minimum size; individual targets
11301 // are responsible for handling scalable vector arguments and
11302 // return values.
11303 ISD::InputArg MyFlags(
11304 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11305 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11306 if (NumRegs > 1 && i == 0)
11307 MyFlags.Flags.setSplit();
11308 // if it isn't first piece, alignment must be 1
11309 else if (i > 0) {
11310 MyFlags.Flags.setOrigAlign(Align(1));
11311 if (i == NumRegs - 1)
11312 MyFlags.Flags.setSplitEnd();
11313 }
11314 Ins.push_back(MyFlags);
11315 }
11316 if (NeedsRegBlock && Value == NumValues - 1)
11317 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11318 PartBase += VT.getStoreSize().getKnownMinValue();
11319 }
11320 }
11321
11322 // Call the target to set up the argument values.
11324 SDValue NewRoot = TLI->LowerFormalArguments(
11325 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11326
11327 // Verify that the target's LowerFormalArguments behaved as expected.
11328 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11329 "LowerFormalArguments didn't return a valid chain!");
11330 assert(InVals.size() == Ins.size() &&
11331 "LowerFormalArguments didn't emit the correct number of values!");
11332 LLVM_DEBUG({
11333 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11334 assert(InVals[i].getNode() &&
11335 "LowerFormalArguments emitted a null value!");
11336 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11337 "LowerFormalArguments emitted a value with the wrong type!");
11338 }
11339 });
11340
11341 // Update the DAG with the new chain value resulting from argument lowering.
11342 DAG.setRoot(NewRoot);
11343
11344 // Set up the argument values.
11345 unsigned i = 0;
11346 if (!FuncInfo->CanLowerReturn) {
11347 // Create a virtual register for the sret pointer, and put in a copy
11348 // from the sret argument into it.
11349 SmallVector<EVT, 1> ValueVTs;
11351 PointerType::get(F.getContext(),
11353 ValueVTs);
11354 MVT VT = ValueVTs[0].getSimpleVT();
11355 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11356 std::optional<ISD::NodeType> AssertOp;
11357 SDValue ArgValue =
11358 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11359 F.getCallingConv(), AssertOp);
11360
11361 MachineFunction& MF = SDB->DAG.getMachineFunction();
11363 Register SRetReg =
11364 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11365 FuncInfo->DemoteRegister = SRetReg;
11366 NewRoot =
11367 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11368 DAG.setRoot(NewRoot);
11369
11370 // i indexes lowered arguments. Bump it past the hidden sret argument.
11371 ++i;
11372 }
11373
11375 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11376 for (const Argument &Arg : F.args()) {
11377 SmallVector<SDValue, 4> ArgValues;
11378 SmallVector<EVT, 4> ValueVTs;
11379 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11380 unsigned NumValues = ValueVTs.size();
11381 if (NumValues == 0)
11382 continue;
11383
11384 bool ArgHasUses = !Arg.use_empty();
11385
11386 // Elide the copying store if the target loaded this argument from a
11387 // suitable fixed stack object.
11388 if (Ins[i].Flags.isCopyElisionCandidate()) {
11389 unsigned NumParts = 0;
11390 for (EVT VT : ValueVTs)
11392 F.getCallingConv(), VT);
11393
11394 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11395 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11396 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11397 }
11398
11399 // If this argument is unused then remember its value. It is used to generate
11400 // debugging information.
11401 bool isSwiftErrorArg =
11403 Arg.hasAttribute(Attribute::SwiftError);
11404 if (!ArgHasUses && !isSwiftErrorArg) {
11405 SDB->setUnusedArgValue(&Arg, InVals[i]);
11406
11407 // Also remember any frame index for use in FastISel.
11408 if (FrameIndexSDNode *FI =
11409 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
11410 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11411 }
11412
11413 for (unsigned Val = 0; Val != NumValues; ++Val) {
11414 EVT VT = ValueVTs[Val];
11416 F.getCallingConv(), VT);
11417 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11418 *CurDAG->getContext(), F.getCallingConv(), VT);
11419
11420 // Even an apparent 'unused' swifterror argument needs to be returned. So
11421 // we do generate a copy for it that can be used on return from the
11422 // function.
11423 if (ArgHasUses || isSwiftErrorArg) {
11424 std::optional<ISD::NodeType> AssertOp;
11425 if (Arg.hasAttribute(Attribute::SExt))
11426 AssertOp = ISD::AssertSext;
11427 else if (Arg.hasAttribute(Attribute::ZExt))
11428 AssertOp = ISD::AssertZext;
11429
11430 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
11431 PartVT, VT, nullptr, NewRoot,
11432 F.getCallingConv(), AssertOp));
11433 }
11434
11435 i += NumParts;
11436 }
11437
11438 // We don't need to do anything else for unused arguments.
11439 if (ArgValues.empty())
11440 continue;
11441
11442 // Note down frame index.
11443 if (FrameIndexSDNode *FI =
11444 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11445 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11446
11447 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11448 SDB->getCurSDLoc());
11449
11450 SDB->setValue(&Arg, Res);
11452 // We want to associate the argument with the frame index, among
11453 // involved operands, that correspond to the lowest address. The
11454 // getCopyFromParts function, called earlier, is swapping the order of
11455 // the operands to BUILD_PAIR depending on endianness. The result of
11456 // that swapping is that the least significant bits of the argument will
11457 // be in the first operand of the BUILD_PAIR node, and the most
11458 // significant bits will be in the second operand.
11459 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11460 if (LoadSDNode *LNode =
11461 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11462 if (FrameIndexSDNode *FI =
11463 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11464 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11465 }
11466
11467 // Analyses past this point are naive and don't expect an assertion.
11468 if (Res.getOpcode() == ISD::AssertZext)
11469 Res = Res.getOperand(0);
11470
11471 // Update the SwiftErrorVRegDefMap.
11472 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11473 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11476 Reg);
11477 }
11478
11479 // If this argument is live outside of the entry block, insert a copy from
11480 // wherever we got it to the vreg that other BB's will reference it as.
11481 if (Res.getOpcode() == ISD::CopyFromReg) {
11482 // If we can, though, try to skip creating an unnecessary vreg.
11483 // FIXME: This isn't very clean... it would be nice to make this more
11484 // general.
11485 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11486 if (Register::isVirtualRegister(Reg)) {
11487 FuncInfo->ValueMap[&Arg] = Reg;
11488 continue;
11489 }
11490 }
11492 FuncInfo->InitializeRegForValue(&Arg);
11493 SDB->CopyToExportRegsIfNeeded(&Arg);
11494 }
11495 }
11496
11497 if (!Chains.empty()) {
11498 Chains.push_back(NewRoot);
11499 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11500 }
11501
11502 DAG.setRoot(NewRoot);
11503
11504 assert(i == InVals.size() && "Argument register count mismatch!");
11505
11506 // If any argument copy elisions occurred and we have debug info, update the
11507 // stale frame indices used in the dbg.declare variable info table.
11508 if (!ArgCopyElisionFrameIndexMap.empty()) {
11511 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11512 if (I != ArgCopyElisionFrameIndexMap.end())
11513 VI.updateStackSlot(I->second);
11514 }
11515 }
11516
11517 // Finally, if the target has anything special to do, allow it to do so.
11519}
11520
11521/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11522/// ensure constants are generated when needed. Remember the virtual registers
11523/// that need to be added to the Machine PHI nodes as input. We cannot just
11524/// directly add them, because expansion might result in multiple MBB's for one
11525/// BB. As such, the start of the BB might correspond to a different MBB than
11526/// the end.
11527void
11528SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11530
11532
11533 // Check PHI nodes in successors that expect a value to be available from this
11534 // block.
11535 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
11536 if (!isa<PHINode>(SuccBB->begin())) continue;
11537 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
11538
11539 // If this terminator has multiple identical successors (common for
11540 // switches), only handle each succ once.
11541 if (!SuccsHandled.insert(SuccMBB).second)
11542 continue;
11543
11545
11546 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11547 // nodes and Machine PHI nodes, but the incoming operands have not been
11548 // emitted yet.
11549 for (const PHINode &PN : SuccBB->phis()) {
11550 // Ignore dead phi's.
11551 if (PN.use_empty())
11552 continue;
11553
11554 // Skip empty types
11555 if (PN.getType()->isEmptyTy())
11556 continue;
11557
11558 unsigned Reg;
11559 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
11560
11561 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
11562 unsigned &RegOut = ConstantsOut[C];
11563 if (RegOut == 0) {
11564 RegOut = FuncInfo.CreateRegs(C);
11565 // We need to zero/sign extend ConstantInt phi operands to match
11566 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
11567 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
11568 if (auto *CI = dyn_cast<ConstantInt>(C))
11569 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
11571 CopyValueToVirtualRegister(C, RegOut, ExtendType);
11572 }
11573 Reg = RegOut;
11574 } else {
11576 FuncInfo.ValueMap.find(PHIOp);
11577 if (I != FuncInfo.ValueMap.end())
11578 Reg = I->second;
11579 else {
11580 assert(isa<AllocaInst>(PHIOp) &&
11581 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
11582 "Didn't codegen value into a register!??");
11583 Reg = FuncInfo.CreateRegs(PHIOp);
11584 CopyValueToVirtualRegister(PHIOp, Reg);
11585 }
11586 }
11587
11588 // Remember that this register needs to added to the machine PHI node as
11589 // the input for this MBB.
11590 SmallVector<EVT, 4> ValueVTs;
11591 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
11592 for (EVT VT : ValueVTs) {
11593 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
11594 for (unsigned i = 0; i != NumRegisters; ++i)
11595 FuncInfo.PHINodesToUpdate.push_back(
11596 std::make_pair(&*MBBI++, Reg + i));
11597 Reg += NumRegisters;
11598 }
11599 }
11600 }
11601
11602 ConstantsOut.clear();
11603}
11604
11605MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
11607 if (++I == FuncInfo.MF->end())
11608 return nullptr;
11609 return &*I;
11610}
11611
11612/// During lowering new call nodes can be created (such as memset, etc.).
11613/// Those will become new roots of the current DAG, but complications arise
11614/// when they are tail calls. In such cases, the call lowering will update
11615/// the root, but the builder still needs to know that a tail call has been
11616/// lowered in order to avoid generating an additional return.
11617void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
11618 // If the node is null, we do have a tail call.
11619 if (MaybeTC.getNode() != nullptr)
11620 DAG.setRoot(MaybeTC);
11621 else
11622 HasTailCall = true;
11623}
11624
11625void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
11626 MachineBasicBlock *SwitchMBB,
11627 MachineBasicBlock *DefaultMBB) {
11628 MachineFunction *CurMF = FuncInfo.MF;
11629 MachineBasicBlock *NextMBB = nullptr;
11631 if (++BBI != FuncInfo.MF->end())
11632 NextMBB = &*BBI;
11633
11634 unsigned Size = W.LastCluster - W.FirstCluster + 1;
11635
11637
11638 if (Size == 2 && W.MBB == SwitchMBB) {
11639 // If any two of the cases has the same destination, and if one value
11640 // is the same as the other, but has one bit unset that the other has set,
11641 // use bit manipulation to do two compares at once. For example:
11642 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
11643 // TODO: This could be extended to merge any 2 cases in switches with 3
11644 // cases.
11645 // TODO: Handle cases where W.CaseBB != SwitchBB.
11646 CaseCluster &Small = *W.FirstCluster;
11647 CaseCluster &Big = *W.LastCluster;
11648
11649 if (Small.Low == Small.High && Big.Low == Big.High &&
11650 Small.MBB == Big.MBB) {
11651 const APInt &SmallValue = Small.Low->getValue();
11652 const APInt &BigValue = Big.Low->getValue();
11653
11654 // Check that there is only one bit different.
11655 APInt CommonBit = BigValue ^ SmallValue;
11656 if (CommonBit.isPowerOf2()) {
11657 SDValue CondLHS = getValue(Cond);
11658 EVT VT = CondLHS.getValueType();
11659 SDLoc DL = getCurSDLoc();
11660
11661 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
11662 DAG.getConstant(CommonBit, DL, VT));
11664 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
11665 ISD::SETEQ);
11666
11667 // Update successor info.
11668 // Both Small and Big will jump to Small.BB, so we sum up the
11669 // probabilities.
11670 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
11671 if (BPI)
11672 addSuccessorWithProb(
11673 SwitchMBB, DefaultMBB,
11674 // The default destination is the first successor in IR.
11675 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
11676 else
11677 addSuccessorWithProb(SwitchMBB, DefaultMBB);
11678
11679 // Insert the true branch.
11680 SDValue BrCond =
11681 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
11682 DAG.getBasicBlock(Small.MBB));
11683 // Insert the false branch.
11684 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
11685 DAG.getBasicBlock(DefaultMBB));
11686
11687 DAG.setRoot(BrCond);
11688 return;
11689 }
11690 }
11691 }
11692
11693 if (TM.getOptLevel() != CodeGenOptLevel::None) {
11694 // Here, we order cases by probability so the most likely case will be
11695 // checked first. However, two clusters can have the same probability in
11696 // which case their relative ordering is non-deterministic. So we use Low
11697 // as a tie-breaker as clusters are guaranteed to never overlap.
11698 llvm::sort(W.FirstCluster, W.LastCluster + 1,
11699 [](const CaseCluster &a, const CaseCluster &b) {
11700 return a.Prob != b.Prob ?
11701 a.Prob > b.Prob :
11702 a.Low->getValue().slt(b.Low->getValue());
11703 });
11704
11705 // Rearrange the case blocks so that the last one falls through if possible
11706 // without changing the order of probabilities.
11707 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
11708 --I;
11709 if (I->Prob > W.LastCluster->Prob)
11710 break;
11711 if (I->Kind == CC_Range && I->MBB == NextMBB) {
11712 std::swap(*I, *W.LastCluster);
11713 break;
11714 }
11715 }
11716 }
11717
11718 // Compute total probability.
11719 BranchProbability DefaultProb = W.DefaultProb;
11720 BranchProbability UnhandledProbs = DefaultProb;
11721 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
11722 UnhandledProbs += I->Prob;
11723
11724 MachineBasicBlock *CurMBB = W.MBB;
11725 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
11726 bool FallthroughUnreachable = false;
11727 MachineBasicBlock *Fallthrough;
11728 if (I == W.LastCluster) {
11729 // For the last cluster, fall through to the default destination.
11730 Fallthrough = DefaultMBB;
11731 FallthroughUnreachable = isa<UnreachableInst>(
11732 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
11733 } else {
11734 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
11735 CurMF->insert(BBI, Fallthrough);
11736 // Put Cond in a virtual register to make it available from the new blocks.
11738 }
11739 UnhandledProbs -= I->Prob;
11740
11741 switch (I->Kind) {
11742 case CC_JumpTable: {
11743 // FIXME: Optimize away range check based on pivot comparisons.
11744 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
11745 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
11746
11747 // The jump block hasn't been inserted yet; insert it here.
11748 MachineBasicBlock *JumpMBB = JT->MBB;
11749 CurMF->insert(BBI, JumpMBB);
11750
11751 auto JumpProb = I->Prob;
11752 auto FallthroughProb = UnhandledProbs;
11753
11754 // If the default statement is a target of the jump table, we evenly
11755 // distribute the default probability to successors of CurMBB. Also
11756 // update the probability on the edge from JumpMBB to Fallthrough.
11757 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
11758 SE = JumpMBB->succ_end();
11759 SI != SE; ++SI) {
11760 if (*SI == DefaultMBB) {
11761 JumpProb += DefaultProb / 2;
11762 FallthroughProb -= DefaultProb / 2;
11763 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
11764 JumpMBB->normalizeSuccProbs();
11765 break;
11766 }
11767 }
11768
11769 // If the default clause is unreachable, propagate that knowledge into
11770 // JTH->FallthroughUnreachable which will use it to suppress the range
11771 // check.
11772 //
11773 // However, don't do this if we're doing branch target enforcement,
11774 // because a table branch _without_ a range check can be a tempting JOP
11775 // gadget - out-of-bounds inputs that are impossible in correct
11776 // execution become possible again if an attacker can influence the
11777 // control flow. So if an attacker doesn't already have a BTI bypass
11778 // available, we don't want them to be able to get one out of this
11779 // table branch.
11780 if (FallthroughUnreachable) {
11781 Function &CurFunc = CurMF->getFunction();
11782 bool HasBranchTargetEnforcement = false;
11783 if (CurFunc.hasFnAttribute("branch-target-enforcement")) {
11784 HasBranchTargetEnforcement =
11785 CurFunc.getFnAttribute("branch-target-enforcement")
11786 .getValueAsBool();
11787 } else {
11788 HasBranchTargetEnforcement =
11789 CurMF->getMMI().getModule()->getModuleFlag(
11790 "branch-target-enforcement");
11791 }
11792 if (!HasBranchTargetEnforcement)
11793 JTH->FallthroughUnreachable = true;
11794 }
11795
11796 if (!JTH->FallthroughUnreachable)
11797 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
11798 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
11799 CurMBB->normalizeSuccProbs();
11800
11801 // The jump table header will be inserted in our current block, do the
11802 // range check, and fall through to our fallthrough block.
11803 JTH->HeaderBB = CurMBB;
11804 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
11805
11806 // If we're in the right place, emit the jump table header right now.
11807 if (CurMBB == SwitchMBB) {
11808 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
11809 JTH->Emitted = true;
11810 }
11811 break;
11812 }
11813 case CC_BitTests: {
11814 // FIXME: Optimize away range check based on pivot comparisons.
11815 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
11816
11817 // The bit test blocks haven't been inserted yet; insert them here.
11818 for (BitTestCase &BTC : BTB->Cases)
11819 CurMF->insert(BBI, BTC.ThisBB);
11820
11821 // Fill in fields of the BitTestBlock.
11822 BTB->Parent = CurMBB;
11823 BTB->Default = Fallthrough;
11824
11825 BTB->DefaultProb = UnhandledProbs;
11826 // If the cases in bit test don't form a contiguous range, we evenly
11827 // distribute the probability on the edge to Fallthrough to two
11828 // successors of CurMBB.
11829 if (!BTB->ContiguousRange) {
11830 BTB->Prob += DefaultProb / 2;
11831 BTB->DefaultProb -= DefaultProb / 2;
11832 }
11833
11834 if (FallthroughUnreachable)
11835 BTB->FallthroughUnreachable = true;
11836
11837 // If we're in the right place, emit the bit test header right now.
11838 if (CurMBB == SwitchMBB) {
11839 visitBitTestHeader(*BTB, SwitchMBB);
11840 BTB->Emitted = true;
11841 }
11842 break;
11843 }
11844 case CC_Range: {
11845 const Value *RHS, *LHS, *MHS;
11847 if (I->Low == I->High) {
11848 // Check Cond == I->Low.
11849 CC = ISD::SETEQ;
11850 LHS = Cond;
11851 RHS=I->Low;
11852 MHS = nullptr;
11853 } else {
11854 // Check I->Low <= Cond <= I->High.
11855 CC = ISD::SETLE;
11856 LHS = I->Low;
11857 MHS = Cond;
11858 RHS = I->High;
11859 }
11860
11861 // If Fallthrough is unreachable, fold away the comparison.
11862 if (FallthroughUnreachable)
11863 CC = ISD::SETTRUE;
11864
11865 // The false probability is the sum of all unhandled cases.
11866 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
11867 getCurSDLoc(), I->Prob, UnhandledProbs);
11868
11869 if (CurMBB == SwitchMBB)
11870 visitSwitchCase(CB, SwitchMBB);
11871 else
11872 SL->SwitchCases.push_back(CB);
11873
11874 break;
11875 }
11876 }
11877 CurMBB = Fallthrough;
11878 }
11879}
11880
11881void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11882 const SwitchWorkListItem &W,
11883 Value *Cond,
11884 MachineBasicBlock *SwitchMBB) {
11885 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11886 "Clusters not sorted?");
11887 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11888
11889 auto [LastLeft, FirstRight, LeftProb, RightProb] =
11890 SL->computeSplitWorkItemInfo(W);
11891
11892 // Use the first element on the right as pivot since we will make less-than
11893 // comparisons against it.
11894 CaseClusterIt PivotCluster = FirstRight;
11895 assert(PivotCluster > W.FirstCluster);
11896 assert(PivotCluster <= W.LastCluster);
11897
11898 CaseClusterIt FirstLeft = W.FirstCluster;
11899 CaseClusterIt LastRight = W.LastCluster;
11900
11901 const ConstantInt *Pivot = PivotCluster->Low;
11902
11903 // New blocks will be inserted immediately after the current one.
11905 ++BBI;
11906
11907 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11908 // we can branch to its destination directly if it's squeezed exactly in
11909 // between the known lower bound and Pivot - 1.
11910 MachineBasicBlock *LeftMBB;
11911 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11912 FirstLeft->Low == W.GE &&
11913 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11914 LeftMBB = FirstLeft->MBB;
11915 } else {
11916 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11917 FuncInfo.MF->insert(BBI, LeftMBB);
11918 WorkList.push_back(
11919 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11920 // Put Cond in a virtual register to make it available from the new blocks.
11922 }
11923
11924 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11925 // single cluster, RHS.Low == Pivot, and we can branch to its destination
11926 // directly if RHS.High equals the current upper bound.
11927 MachineBasicBlock *RightMBB;
11928 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11929 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11930 RightMBB = FirstRight->MBB;
11931 } else {
11932 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11933 FuncInfo.MF->insert(BBI, RightMBB);
11934 WorkList.push_back(
11935 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11936 // Put Cond in a virtual register to make it available from the new blocks.
11938 }
11939
11940 // Create the CaseBlock record that will be used to lower the branch.
11941 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11942 getCurSDLoc(), LeftProb, RightProb);
11943
11944 if (W.MBB == SwitchMBB)
11945 visitSwitchCase(CB, SwitchMBB);
11946 else
11947 SL->SwitchCases.push_back(CB);
11948}
11949
11950// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11951// from the swith statement.
11953 BranchProbability PeeledCaseProb) {
11954 if (PeeledCaseProb == BranchProbability::getOne())
11956 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11957
11958 uint32_t Numerator = CaseProb.getNumerator();
11959 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11960 return BranchProbability(Numerator, std::max(Numerator, Denominator));
11961}
11962
11963// Try to peel the top probability case if it exceeds the threshold.
11964// Return current MachineBasicBlock for the switch statement if the peeling
11965// does not occur.
11966// If the peeling is performed, return the newly created MachineBasicBlock
11967// for the peeled switch statement. Also update Clusters to remove the peeled
11968// case. PeeledCaseProb is the BranchProbability for the peeled case.
11969MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11970 const SwitchInst &SI, CaseClusterVector &Clusters,
11971 BranchProbability &PeeledCaseProb) {
11972 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11973 // Don't perform if there is only one cluster or optimizing for size.
11974 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11976 SwitchMBB->getParent()->getFunction().hasMinSize())
11977 return SwitchMBB;
11978
11980 unsigned PeeledCaseIndex = 0;
11981 bool SwitchPeeled = false;
11982 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11983 CaseCluster &CC = Clusters[Index];
11984 if (CC.Prob < TopCaseProb)
11985 continue;
11986 TopCaseProb = CC.Prob;
11987 PeeledCaseIndex = Index;
11988 SwitchPeeled = true;
11989 }
11990 if (!SwitchPeeled)
11991 return SwitchMBB;
11992
11993 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11994 << TopCaseProb << "\n");
11995
11996 // Record the MBB for the peeled switch statement.
11997 MachineFunction::iterator BBI(SwitchMBB);
11998 ++BBI;
11999 MachineBasicBlock *PeeledSwitchMBB =
12001 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
12002
12003 ExportFromCurrentBlock(SI.getCondition());
12004 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12005 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12006 nullptr, nullptr, TopCaseProb.getCompl()};
12007 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12008
12009 Clusters.erase(PeeledCaseIt);
12010 for (CaseCluster &CC : Clusters) {
12011 LLVM_DEBUG(
12012 dbgs() << "Scale the probablity for one cluster, before scaling: "
12013 << CC.Prob << "\n");
12014 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12015 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12016 }
12017 PeeledCaseProb = TopCaseProb;
12018 return PeeledSwitchMBB;
12019}
12020
12021void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12022 // Extract cases from the switch.
12024 CaseClusterVector Clusters;
12025 Clusters.reserve(SI.getNumCases());
12026 for (auto I : SI.cases()) {
12027 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
12028 const ConstantInt *CaseVal = I.getCaseValue();
12029 BranchProbability Prob =
12030 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12031 : BranchProbability(1, SI.getNumCases() + 1);
12032 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12033 }
12034
12035 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
12036
12037 // Cluster adjacent cases with the same destination. We do this at all
12038 // optimization levels because it's cheap to do and will make codegen faster
12039 // if there are many clusters.
12040 sortAndRangeify(Clusters);
12041
12042 // The branch probablity of the peeled case.
12044 MachineBasicBlock *PeeledSwitchMBB =
12045 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12046
12047 // If there is only the default destination, jump there directly.
12048 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12049 if (Clusters.empty()) {
12050 assert(PeeledSwitchMBB == SwitchMBB);
12051 SwitchMBB->addSuccessor(DefaultMBB);
12052 if (DefaultMBB != NextBlock(SwitchMBB)) {
12053 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12054 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12055 }
12056 return;
12057 }
12058
12059 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12060 DAG.getBFI());
12061 SL->findBitTestClusters(Clusters, &SI);
12062
12063 LLVM_DEBUG({
12064 dbgs() << "Case clusters: ";
12065 for (const CaseCluster &C : Clusters) {
12066 if (C.Kind == CC_JumpTable)
12067 dbgs() << "JT:";
12068 if (C.Kind == CC_BitTests)
12069 dbgs() << "BT:";
12070
12071 C.Low->getValue().print(dbgs(), true);
12072 if (C.Low != C.High) {
12073 dbgs() << '-';
12074 C.High->getValue().print(dbgs(), true);
12075 }
12076 dbgs() << ' ';
12077 }
12078 dbgs() << '\n';
12079 });
12080
12081 assert(!Clusters.empty());
12082 SwitchWorkList WorkList;
12083 CaseClusterIt First = Clusters.begin();
12084 CaseClusterIt Last = Clusters.end() - 1;
12085 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12086 // Scale the branchprobability for DefaultMBB if the peel occurs and
12087 // DefaultMBB is not replaced.
12088 if (PeeledCaseProb != BranchProbability::getZero() &&
12089 DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
12090 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12091 WorkList.push_back(
12092 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12093
12094 while (!WorkList.empty()) {
12095 SwitchWorkListItem W = WorkList.pop_back_val();
12096 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12097
12098 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12099 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12100 // For optimized builds, lower large range as a balanced binary tree.
12101 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12102 continue;
12103 }
12104
12105 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12106 }
12107}
12108
12109void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12111 auto DL = getCurSDLoc();
12112 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12113 setValue(&I, DAG.getStepVector(DL, ResultVT));
12114}
12115
12116void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12118 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12119
12120 SDLoc DL = getCurSDLoc();
12121 SDValue V = getValue(I.getOperand(0));
12122 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12123
12124 if (VT.isScalableVector()) {
12126 return;
12127 }
12128
12129 // Use VECTOR_SHUFFLE for the fixed-length vector
12130 // to maintain existing behavior.
12132 unsigned NumElts = VT.getVectorMinNumElements();
12133 for (unsigned i = 0; i != NumElts; ++i)
12134 Mask.push_back(NumElts - 1 - i);
12135
12136 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12137}
12138
12139void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
12140 auto DL = getCurSDLoc();
12141 SDValue InVec = getValue(I.getOperand(0));
12142 EVT OutVT =
12144
12145 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12146
12147 // ISD Node needs the input vectors split into two equal parts
12148 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12150 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12151 DAG.getVectorIdxConstant(OutNumElts, DL));
12152
12153 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12154 // legalisation and combines.
12155 if (OutVT.isFixedLengthVector()) {
12156 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12157 createStrideMask(0, 2, OutNumElts));
12158 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12159 createStrideMask(1, 2, OutNumElts));
12160 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12161 setValue(&I, Res);
12162 return;
12163 }
12164
12166 DAG.getVTList(OutVT, OutVT), Lo, Hi);
12167 setValue(&I, Res);
12168}
12169
12170void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
12171 auto DL = getCurSDLoc();
12172 EVT InVT = getValue(I.getOperand(0)).getValueType();
12173 SDValue InVec0 = getValue(I.getOperand(0));
12174 SDValue InVec1 = getValue(I.getOperand(1));
12176 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12177
12178 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12179 // legalisation and combines.
12180 if (OutVT.isFixedLengthVector()) {
12181 unsigned NumElts = InVT.getVectorMinNumElements();
12182 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
12183 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12184 createInterleaveMask(NumElts, 2)));
12185 return;
12186 }
12187
12189 DAG.getVTList(InVT, InVT), InVec0, InVec1);
12190 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
12191 Res.getValue(1));
12192 setValue(&I, Res);
12193}
12194
12195void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12196 SmallVector<EVT, 4> ValueVTs;
12198 ValueVTs);
12199 unsigned NumValues = ValueVTs.size();
12200 if (NumValues == 0) return;
12201
12202 SmallVector<SDValue, 4> Values(NumValues);
12203 SDValue Op = getValue(I.getOperand(0));
12204
12205 for (unsigned i = 0; i != NumValues; ++i)
12206 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12207 SDValue(Op.getNode(), Op.getResNo() + i));
12208
12210 DAG.getVTList(ValueVTs), Values));
12211}
12212
12213void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12215 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12216
12217 SDLoc DL = getCurSDLoc();
12218 SDValue V1 = getValue(I.getOperand(0));
12219 SDValue V2 = getValue(I.getOperand(1));
12220 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12221
12222 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12223 if (VT.isScalableVector()) {
12224 MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
12225 setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12226 DAG.getConstant(Imm, DL, IdxVT)));
12227 return;
12228 }
12229
12230 unsigned NumElts = VT.getVectorNumElements();
12231
12232 uint64_t Idx = (NumElts + Imm) % NumElts;
12233
12234 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12236 for (unsigned i = 0; i < NumElts; ++i)
12237 Mask.push_back(Idx + i);
12238 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12239}
12240
12241// Consider the following MIR after SelectionDAG, which produces output in
12242// phyregs in the first case or virtregs in the second case.
12243//
12244// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12245// %5:gr32 = COPY $ebx
12246// %6:gr32 = COPY $edx
12247// %1:gr32 = COPY %6:gr32
12248// %0:gr32 = COPY %5:gr32
12249//
12250// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12251// %1:gr32 = COPY %6:gr32
12252// %0:gr32 = COPY %5:gr32
12253//
12254// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12255// Given %1, we'd like to return $edx in the first case and %6 in the second.
12256//
12257// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12258// to a single virtreg (such as %0). The remaining outputs monotonically
12259// increase in virtreg number from there. If a callbr has no outputs, then it
12260// should not have a corresponding callbr landingpad; in fact, the callbr
12261// landingpad would not even be able to refer to such a callbr.
12263 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12264 // There is definitely at least one copy.
12265 assert(MI->getOpcode() == TargetOpcode::COPY &&
12266 "start of copy chain MUST be COPY");
12267 Reg = MI->getOperand(1).getReg();
12268 MI = MRI.def_begin(Reg)->getParent();
12269 // There may be an optional second copy.
12270 if (MI->getOpcode() == TargetOpcode::COPY) {
12271 assert(Reg.isVirtual() && "expected COPY of virtual register");
12272 Reg = MI->getOperand(1).getReg();
12273 assert(Reg.isPhysical() && "expected COPY of physical register");
12274 MI = MRI.def_begin(Reg)->getParent();
12275 }
12276 // The start of the chain must be an INLINEASM_BR.
12277 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12278 "end of copy chain MUST be INLINEASM_BR");
12279 return Reg;
12280}
12281
12282// We must do this walk rather than the simpler
12283// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12284// otherwise we will end up with copies of virtregs only valid along direct
12285// edges.
12286void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12287 SmallVector<EVT, 8> ResultVTs;
12288 SmallVector<SDValue, 8> ResultValues;
12289 const auto *CBR =
12290 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12291
12295
12296 unsigned InitialDef = FuncInfo.ValueMap[CBR];
12297 SDValue Chain = DAG.getRoot();
12298
12299 // Re-parse the asm constraints string.
12300 TargetLowering::AsmOperandInfoVector TargetConstraints =
12301 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12302 for (auto &T : TargetConstraints) {
12303 SDISelAsmOperandInfo OpInfo(T);
12304 if (OpInfo.Type != InlineAsm::isOutput)
12305 continue;
12306
12307 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12308 // individual constraint.
12309 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12310
12311 switch (OpInfo.ConstraintType) {
12314 // Fill in OpInfo.AssignedRegs.Regs.
12315 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12316
12317 // getRegistersForValue may produce 1 to many registers based on whether
12318 // the OpInfo.ConstraintVT is legal on the target or not.
12319 for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
12320 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12321 if (Register::isPhysicalRegister(OriginalDef))
12322 FuncInfo.MBB->addLiveIn(OriginalDef);
12323 // Update the assigned registers to use the original defs.
12324 OpInfo.AssignedRegs.Regs[i] = OriginalDef;
12325 }
12326
12327 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12328 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12329 ResultValues.push_back(V);
12330 ResultVTs.push_back(OpInfo.ConstraintVT);
12331 break;
12332 }
12334 SDValue Flag;
12335 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12336 OpInfo, DAG);
12337 ++InitialDef;
12338 ResultValues.push_back(V);
12339 ResultVTs.push_back(OpInfo.ConstraintVT);
12340 break;
12341 }
12342 default:
12343 break;
12344 }
12345 }
12347 DAG.getVTList(ResultVTs), ResultValues);
12348 setValue(&I, V);
12349}
unsigned const MachineRegisterInfo * MRI
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:943
#define Check(C,...)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
uint64_t High
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< unsigned, TypeSize > > &Regs, const SDValue &N)
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
an instruction to allocate memory on the stack
Definition: Instructions.h:59
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition: Function.cpp:338
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:49
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
bool getValueAsBool() const
Return the attribute's value as a boolean.
Definition: Attributes.cpp:335
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:349
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:553
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:368
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
const Instruction & back() const
Definition: BasicBlock.h:454
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
Definition: BitVector.h:461
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
The address of a basic block.
Definition: Constants.h:889
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1461
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2367
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1767
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1629
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2343
Value * getCalledOperand() const
Definition: InstrTypes.h:1702
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1654
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1635
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2251
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1567
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
unsigned arg_size() const
Definition: InstrTypes.h:1652
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1786
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:950
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:960
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:583
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1017
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2140
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:268
This is the shared class of boolean and integer constants.
Definition: Constants.h:80
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:205
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:856
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:145
This class represents a range of values.
Definition: ConstantRange.h:47
APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
bool isEmptySet() const
Return true if this set contains no members.
bool isUpperWrapped() const
Return true if the exclusive upper bound wraps around the unsigned domain.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition: Constants.h:507
This is an important base class in LLVM.
Definition: Constant.h:41
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Debug location.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
bool isBigEndian() const
Definition: DataLayout.h:239
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:276
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:420
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition: DebugLoc.h:33
DILocation * getInlinedAt() const
Definition: DebugLoc.cpp:39
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:296
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:302
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:307
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
An instruction for ordering other memory operations.
Definition: Instructions.h:460
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:142
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Type * getReturnType() const
Definition: DerivedTypes.h:124
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:703
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:230
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:330
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:262
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1882
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:338
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
size_t arg_size() const
Definition: Function.h:846
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.h:213
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:677
Garbage collection metadata for a single function.
Definition: GCMetadata.h:78
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:118
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:454
const BasicBlock * getParent() const
Definition: Instruction.h:152
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1707
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Definition: DerivedTypes.h:51
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:321
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
Machine Value Type.
uint64_t getScalarSizeInBits() const
@ INVALID_SIMPLE_VALUE_TYPE
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
const MCContext & getContext() const
const Module * getModule() const
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
unsigned getCurrentCallSite()
Get the call site currently being processed, if any.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition: MapVector.h:163
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition: MapVector.h:118
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition: Module.cpp:331
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:75
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Definition: Operator.h:150
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, const TargetLibraryInfo *li)
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineFunction * MF
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:361
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:722
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:954
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:474
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:488
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:487
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:480
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:773
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:708
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:484
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:878
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual const char * getClearCacheBuiltinName() const
Return the builtin name for the __builtin___clear_cache intrinsic Default is to invoke the clear cach...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:750
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Definition: Triple.h:1005
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:895
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:216
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Base class of all SIMD vector types.
Definition: DerivedTypes.h:403
Type * getElementType() const
Definition: DerivedTypes.h:436
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
Definition: CallingConv.h:245
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:163
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1126
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1122
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ CONVERGENCECTRL_ANCHOR
Definition: ISDOpcodes.h:1390
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1276
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:998
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:147
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1155
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1002
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1021
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:151
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1233
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1025
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FPTRUNC_ROUND
Definition: ISDOpcodes.h:480
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1199
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1274
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:135
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1108
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:885
@ CONVERGENCECTRL_GLUE
Definition: ISDOpcodes.h:1396
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ PREALLOCATED_SETUP
Definition: ISDOpcodes.h:1160
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1188
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:101
@ CONVERGENCECTRL_ENTRY
Definition: ISDOpcodes.h:1391
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1047
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1275
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ PREALLOCATED_ARG
Definition: ISDOpcodes.h:1163
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1052
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1056
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:586
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1151
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1277
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1225
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:880
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1117
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1016
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:993
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1211
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1329
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:114
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1215
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1041
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:591
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:1174
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:1097
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:129
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1279
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1182
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1208
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:106
@ CONVERGENCECTRL_LOOP
Definition: ISDOpcodes.h:1392
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1094
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:141
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1070
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1113
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1278
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:580
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1320
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1205
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1485
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
@ ReallyHidden
Definition: CommandLine.h:139
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebStrict
This corresponds to "fpexcept.strict".
Definition: FPEnv.h:41
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition: FPEnv.h:40
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
constexpr float log2ef
Definition: MathExtras.h:50
constexpr double e
Definition: MathExtras.h:31
constexpr float ln2f
Definition: MathExtras.h:48
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
@ Length
Definition: DWP.cpp:456
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1689
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
@ Done
Definition: Threading.h:61
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
gep_type_iterator gep_type_end(const User *GEP)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition: STLExtras.h:885
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition: STLExtras.h:322
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition: Analysis.cpp:199
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2534
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2392
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition: Analysis.cpp:221
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2060
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2048
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:428
unsigned succ_size(const MachineBasicBlock *BB)
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
#define NC
Definition: regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:249
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
uint64_t getScalarStoreSize() const
Definition: ValueTypes.h:387
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:366
bool isScalableVT() const
Return true if the type is a scalable type.
Definition: ValueTypes.h:183
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:101
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:244
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< unsigned, 4 > Regs
This list holds the registers assigned to the values.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< std::pair< unsigned, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)