LLVM 17.0.0git
FastISel.cpp
Go to the documentation of this file.
1//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the FastISel class.
10//
11// "Fast" instruction selection is designed to emit very poor code quickly.
12// Also, it is not designed to be able to do much lowering, so most illegal
13// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14// also not intended to be able to do much optimization, except in a few cases
15// where doing optimizations reduces overall compile time. For example, folding
16// constants into immediate fields is often done, because it's cheap and it
17// reduces the number of instructions later phases have to examine.
18//
19// "Fast" instruction selection is able to fail gracefully and transfer
20// control to the SelectionDAG selector for operations that it doesn't
21// support. In many cases, this allows us to avoid duplicating a lot of
22// the complicated lowering logic that SelectionDAG currently has.
23//
24// The intended use for "fast" instruction selection is "-O0" mode
25// compilation, where the quality of the generated code is irrelevant when
26// weighed against the speed at which the code can be generated. Also,
27// at -O0, the LLVM optimizers are not running, and this makes the
28// compile time of codegen a much higher portion of the overall compile
29// time. Despite its limitations, "fast" instruction selection is able to
30// handle enough code on its own to provide noticeable overall speedups
31// in -O0 compiles.
32//
33// Basic operations are supported in a target-independent way, by reading
34// the same instruction descriptions that the SelectionDAG selector reads,
35// and identifying simple arithmetic operations that can be directly selected
36// from simple operators. More complicated operations currently require
37// target-specific code.
38//
39//===----------------------------------------------------------------------===//
40
42#include "llvm/ADT/APFloat.h"
43#include "llvm/ADT/APSInt.h"
44#include "llvm/ADT/DenseMap.h"
48#include "llvm/ADT/Statistic.h"
67#include "llvm/IR/Argument.h"
68#include "llvm/IR/Attributes.h"
69#include "llvm/IR/BasicBlock.h"
70#include "llvm/IR/CallingConv.h"
71#include "llvm/IR/Constant.h"
72#include "llvm/IR/Constants.h"
73#include "llvm/IR/DataLayout.h"
74#include "llvm/IR/DebugLoc.h"
77#include "llvm/IR/Function.h"
79#include "llvm/IR/GlobalValue.h"
80#include "llvm/IR/InlineAsm.h"
81#include "llvm/IR/InstrTypes.h"
82#include "llvm/IR/Instruction.h"
85#include "llvm/IR/LLVMContext.h"
86#include "llvm/IR/Mangler.h"
87#include "llvm/IR/Metadata.h"
88#include "llvm/IR/Operator.h"
90#include "llvm/IR/Type.h"
91#include "llvm/IR/User.h"
92#include "llvm/IR/Value.h"
93#include "llvm/MC/MCContext.h"
94#include "llvm/MC/MCInstrDesc.h"
96#include "llvm/Support/Debug.h"
103#include <algorithm>
104#include <cassert>
105#include <cstdint>
106#include <iterator>
107#include <optional>
108#include <utility>
109
110using namespace llvm;
111using namespace PatternMatch;
112
113#define DEBUG_TYPE "isel"
114
115STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116 "target-independent selector");
117STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118 "target-specific selector");
119STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121/// Set the current block to which generated machine instructions will be
122/// appended.
124 assert(LocalValueMap.empty() &&
125 "local values should be cleared after finishing a BB");
126
127 // Instructions are appended to FuncInfo.MBB. If the basic block already
128 // contains labels or copies, use the last instruction as the last local
129 // value.
130 EmitStartPt = nullptr;
131 if (!FuncInfo.MBB->empty())
132 EmitStartPt = &FuncInfo.MBB->back();
134}
135
136void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
139 if (!FuncInfo.CanLowerReturn)
140 // Fallback to SDISel argument lowering code to deal with sret pointer
141 // parameter.
142 return false;
143
144 if (!fastLowerArguments())
145 return false;
146
147 // Enter arguments into ValueMap for uses in non-entry BBs.
148 for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
149 E = FuncInfo.Fn->arg_end();
150 I != E; ++I) {
152 assert(VI != LocalValueMap.end() && "Missed an argument?");
153 FuncInfo.ValueMap[&*I] = VI->second;
154 }
155 return true;
156}
157
158/// Return the defined register if this instruction defines exactly one
159/// virtual register and uses no other virtual registers. Otherwise return 0.
161 Register RegDef;
162 for (const MachineOperand &MO : MI.operands()) {
163 if (!MO.isReg())
164 continue;
165 if (MO.isDef()) {
166 if (RegDef)
167 return Register();
168 RegDef = MO.getReg();
169 } else if (MO.getReg().isVirtual()) {
170 // This is another use of a vreg. Don't delete it.
171 return Register();
172 }
173 }
174 return RegDef;
175}
176
177static bool isRegUsedByPhiNodes(Register DefReg,
179 for (auto &P : FuncInfo.PHINodesToUpdate)
180 if (P.second == DefReg)
181 return true;
182 return false;
183}
184
185void FastISel::flushLocalValueMap() {
186 // If FastISel bails out, it could leave local value instructions behind
187 // that aren't used for anything. Detect and erase those.
189 // Save the first instruction after local values, for later.
191 ++FirstNonValue;
192
195 : FuncInfo.MBB->rend();
197 for (MachineInstr &LocalMI :
199 Register DefReg = findLocalRegDef(LocalMI);
200 if (!DefReg)
201 continue;
202 if (FuncInfo.RegsWithFixups.count(DefReg))
203 continue;
204 bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
205 if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
206 if (EmitStartPt == &LocalMI)
208 LLVM_DEBUG(dbgs() << "removing dead local value materialization"
209 << LocalMI);
210 LocalMI.eraseFromParent();
211 }
212 }
213
214 if (FirstNonValue != FuncInfo.MBB->end()) {
215 // See if there are any local value instructions left. If so, we want to
216 // make sure the first one has a debug location; if it doesn't, use the
217 // first non-value instruction's debug location.
218
219 // If EmitStartPt is non-null, this block had copies at the top before
220 // FastISel started doing anything; it points to the last one, so the
221 // first local value instruction is the one after EmitStartPt.
222 // If EmitStartPt is null, the first local value instruction is at the
223 // top of the block.
224 MachineBasicBlock::iterator FirstLocalValue =
226 : FuncInfo.MBB->begin();
227 if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc())
228 FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
229 }
230 }
231
232 LocalValueMap.clear();
235 SavedInsertPt = FuncInfo.InsertPt;
236}
237
239 EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
240 // Don't handle non-simple values in FastISel.
241 if (!RealVT.isSimple())
242 return Register();
243
244 // Ignore illegal types. We must do this before looking up the value
245 // in ValueMap because Arguments are given virtual registers regardless
246 // of whether FastISel can handle them.
247 MVT VT = RealVT.getSimpleVT();
248 if (!TLI.isTypeLegal(VT)) {
249 // Handle integer promotions, though, because they're common and easy.
250 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
251 VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
252 else
253 return Register();
254 }
255
256 // Look up the value to see if we already have a register for it.
258 if (Reg)
259 return Reg;
260
261 // In bottom-up mode, just create the virtual register which will be used
262 // to hold the value. It will be materialized later.
263 if (isa<Instruction>(V) &&
264 (!isa<AllocaInst>(V) ||
265 !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
266 return FuncInfo.InitializeRegForValue(V);
267
268 SavePoint SaveInsertPt = enterLocalValueArea();
269
270 // Materialize the value in a register. Emit any instructions in the
271 // local value area.
272 Reg = materializeRegForValue(V, VT);
273
274 leaveLocalValueArea(SaveInsertPt);
275
276 return Reg;
277}
278
279Register FastISel::materializeConstant(const Value *V, MVT VT) {
280 Register Reg;
281 if (const auto *CI = dyn_cast<ConstantInt>(V)) {
282 if (CI->getValue().getActiveBits() <= 64)
283 Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
284 } else if (isa<AllocaInst>(V))
285 Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
286 else if (isa<ConstantPointerNull>(V))
287 // Translate this as an integer zero so that it can be
288 // local-CSE'd with actual integer zeros.
289 Reg =
291 else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
292 if (CF->isNullValue())
293 Reg = fastMaterializeFloatZero(CF);
294 else
295 // Try to emit the constant directly.
296 Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
297
298 if (!Reg) {
299 // Try to emit the constant by using an integer constant with a cast.
300 const APFloat &Flt = CF->getValueAPF();
301 EVT IntVT = TLI.getPointerTy(DL);
302 uint32_t IntBitWidth = IntVT.getSizeInBits();
303 APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
304 bool isExact;
305 (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
306 if (isExact) {
307 Register IntegerReg =
308 getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
309 if (IntegerReg)
310 Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP,
311 IntegerReg);
312 }
313 }
314 } else if (const auto *Op = dyn_cast<Operator>(V)) {
315 if (!selectOperator(Op, Op->getOpcode()))
316 if (!isa<Instruction>(Op) ||
317 !fastSelectInstruction(cast<Instruction>(Op)))
318 return 0;
320 } else if (isa<UndefValue>(V)) {
322 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
323 TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
324 }
325 return Reg;
326}
327
328/// Helper for getRegForValue. This function is called when the value isn't
329/// already available in a register and must be materialized with new
330/// instructions.
331Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
333 // Give the target-specific code a try first.
334 if (isa<Constant>(V))
335 Reg = fastMaterializeConstant(cast<Constant>(V));
336
337 // If target-specific code couldn't or didn't want to handle the value, then
338 // give target-independent code a try.
339 if (!Reg)
340 Reg = materializeConstant(V, VT);
341
342 // Don't cache constant materializations in the general ValueMap.
343 // To do so would require tracking what uses they dominate.
344 if (Reg) {
347 }
348 return Reg;
349}
350
352 // Look up the value to see if we already have a register for it. We
353 // cache values defined by Instructions across blocks, and other values
354 // only locally. This is because Instructions already have the SSA
355 // def-dominates-use requirement enforced.
357 if (I != FuncInfo.ValueMap.end())
358 return I->second;
359 return LocalValueMap[V];
360}
361
362void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
363 if (!isa<Instruction>(I)) {
364 LocalValueMap[I] = Reg;
365 return;
366 }
367
368 Register &AssignedReg = FuncInfo.ValueMap[I];
369 if (!AssignedReg)
370 // Use the new register.
371 AssignedReg = Reg;
372 else if (Reg != AssignedReg) {
373 // Arrange for uses of AssignedReg to be replaced by uses of Reg.
374 for (unsigned i = 0; i < NumRegs; i++) {
375 FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
376 FuncInfo.RegsWithFixups.insert(Reg + i);
377 }
378
379 AssignedReg = Reg;
380 }
381}
382
385 if (!IdxN)
386 // Unhandled operand. Halt "fast" selection and bail.
387 return Register();
388
389 // If the index is smaller or larger than intptr_t, truncate or extend it.
390 MVT PtrVT = TLI.getPointerTy(DL);
391 EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
392 if (IdxVT.bitsLT(PtrVT)) {
393 IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN);
394 } else if (IdxVT.bitsGT(PtrVT)) {
395 IdxN =
396 fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN);
397 }
398 return IdxN;
399}
400
402 if (getLastLocalValue()) {
403 FuncInfo.InsertPt = getLastLocalValue();
404 FuncInfo.MBB = FuncInfo.InsertPt->getParent();
405 ++FuncInfo.InsertPt;
406 } else
407 FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
408}
409
412 assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
413 "Invalid iterator!");
414 while (I != E) {
415 if (SavedInsertPt == I)
416 SavedInsertPt = E;
417 if (EmitStartPt == I)
418 EmitStartPt = E.isValid() ? &*E : nullptr;
419 if (LastLocalValue == I)
420 LastLocalValue = E.isValid() ? &*E : nullptr;
421
422 MachineInstr *Dead = &*I;
423 ++I;
424 Dead->eraseFromParent();
425 ++NumFastIselDead;
426 }
428}
429
431 SavePoint OldInsertPt = FuncInfo.InsertPt;
433 return OldInsertPt;
434}
435
437 if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
438 LastLocalValue = &*std::prev(FuncInfo.InsertPt);
439
440 // Restore the previous insert position.
441 FuncInfo.InsertPt = OldInsertPt;
442}
443
444bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
445 EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
446 if (VT == MVT::Other || !VT.isSimple())
447 // Unhandled type. Halt "fast" selection and bail.
448 return false;
449
450 // We only handle legal types. For example, on x86-32 the instruction
451 // selector contains all of the 64-bit instructions from x86-64,
452 // under the assumption that i64 won't be used if the target doesn't
453 // support it.
454 if (!TLI.isTypeLegal(VT)) {
455 // MVT::i1 is special. Allow AND, OR, or XOR because they
456 // don't require additional zeroing, which makes them easy.
457 if (VT == MVT::i1 && ISD::isBitwiseLogicOp(ISDOpcode))
458 VT = TLI.getTypeToTransformTo(I->getContext(), VT);
459 else
460 return false;
461 }
462
463 // Check if the first operand is a constant, and handle it as "ri". At -O0,
464 // we don't have anything that canonicalizes operand order.
465 if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
466 if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
467 Register Op1 = getRegForValue(I->getOperand(1));
468 if (!Op1)
469 return false;
470
471 Register ResultReg =
472 fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, CI->getZExtValue(),
473 VT.getSimpleVT());
474 if (!ResultReg)
475 return false;
476
477 // We successfully emitted code for the given LLVM Instruction.
478 updateValueMap(I, ResultReg);
479 return true;
480 }
481
482 Register Op0 = getRegForValue(I->getOperand(0));
483 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
484 return false;
485
486 // Check if the second operand is a constant and handle it appropriately.
487 if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
488 uint64_t Imm = CI->getSExtValue();
489
490 // Transform "sdiv exact X, 8" -> "sra X, 3".
491 if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
492 cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
493 Imm = Log2_64(Imm);
494 ISDOpcode = ISD::SRA;
495 }
496
497 // Transform "urem x, pow2" -> "and x, pow2-1".
498 if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
499 isPowerOf2_64(Imm)) {
500 --Imm;
501 ISDOpcode = ISD::AND;
502 }
503
504 Register ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0, Imm,
505 VT.getSimpleVT());
506 if (!ResultReg)
507 return false;
508
509 // We successfully emitted code for the given LLVM Instruction.
510 updateValueMap(I, ResultReg);
511 return true;
512 }
513
514 Register Op1 = getRegForValue(I->getOperand(1));
515 if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
516 return false;
517
518 // Now we have both operands in registers. Emit the instruction.
519 Register ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
520 ISDOpcode, Op0, Op1);
521 if (!ResultReg)
522 // Target-specific code wasn't able to find a machine opcode for
523 // the given ISD opcode and type. Halt "fast" selection and bail.
524 return false;
525
526 // We successfully emitted code for the given LLVM Instruction.
527 updateValueMap(I, ResultReg);
528 return true;
529}
530
532 Register N = getRegForValue(I->getOperand(0));
533 if (!N) // Unhandled operand. Halt "fast" selection and bail.
534 return false;
535
536 // FIXME: The code below does not handle vector GEPs. Halt "fast" selection
537 // and bail.
538 if (isa<VectorType>(I->getType()))
539 return false;
540
541 // Keep a running tab of the total offset to coalesce multiple N = N + Offset
542 // into a single N = N + TotalOffset.
543 uint64_t TotalOffs = 0;
544 // FIXME: What's a good SWAG number for MaxOffs?
545 uint64_t MaxOffs = 2048;
546 MVT VT = TLI.getPointerTy(DL);
548 GTI != E; ++GTI) {
549 const Value *Idx = GTI.getOperand();
550 if (StructType *StTy = GTI.getStructTypeOrNull()) {
551 uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
552 if (Field) {
553 // N = N + Offset
554 TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
555 if (TotalOffs >= MaxOffs) {
556 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
557 if (!N) // Unhandled operand. Halt "fast" selection and bail.
558 return false;
559 TotalOffs = 0;
560 }
561 }
562 } else {
563 Type *Ty = GTI.getIndexedType();
564
565 // If this is a constant subscript, handle it quickly.
566 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
567 if (CI->isZero())
568 continue;
569 // N = N + Offset
570 uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
571 TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
572 if (TotalOffs >= MaxOffs) {
573 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
574 if (!N) // Unhandled operand. Halt "fast" selection and bail.
575 return false;
576 TotalOffs = 0;
577 }
578 continue;
579 }
580 if (TotalOffs) {
581 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
582 if (!N) // Unhandled operand. Halt "fast" selection and bail.
583 return false;
584 TotalOffs = 0;
585 }
586
587 // N = N + Idx * ElementSize;
588 uint64_t ElementSize = DL.getTypeAllocSize(Ty);
590 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
591 return false;
592
593 if (ElementSize != 1) {
594 IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, ElementSize, VT);
595 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
596 return false;
597 }
598 N = fastEmit_rr(VT, VT, ISD::ADD, N, IdxN);
599 if (!N) // Unhandled operand. Halt "fast" selection and bail.
600 return false;
601 }
602 }
603 if (TotalOffs) {
604 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
605 if (!N) // Unhandled operand. Halt "fast" selection and bail.
606 return false;
607 }
608
609 // We successfully emitted code for the given LLVM Instruction.
611 return true;
612}
613
614bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
615 const CallInst *CI, unsigned StartIdx) {
616 for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
617 Value *Val = CI->getArgOperand(i);
618 // Check for constants and encode them with a StackMaps::ConstantOp prefix.
619 if (const auto *C = dyn_cast<ConstantInt>(Val)) {
620 Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
621 Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
622 } else if (isa<ConstantPointerNull>(Val)) {
623 Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
625 } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
626 // Values coming from a stack location also require a special encoding,
627 // but that is added later on by the target specific frame index
628 // elimination implementation.
629 auto SI = FuncInfo.StaticAllocaMap.find(AI);
630 if (SI != FuncInfo.StaticAllocaMap.end())
632 else
633 return false;
634 } else {
636 if (!Reg)
637 return false;
638 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
639 }
640 }
641 return true;
642}
643
645 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
646 // [live variables...])
647 assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
648 "Stackmap cannot return a value.");
649
650 // The stackmap intrinsic only records the live variables (the arguments
651 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
652 // intrinsic, this won't be lowered to a function call. This means we don't
653 // have to worry about calling conventions and target-specific lowering code.
654 // Instead we perform the call lowering right here.
655 //
656 // CALLSEQ_START(0, 0...)
657 // STACKMAP(id, nbytes, ...)
658 // CALLSEQ_END(0, 0)
659 //
661
662 // Add the <id> and <numBytes> constants.
663 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
664 "Expected a constant integer.");
665 const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
666 Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
667
668 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
669 "Expected a constant integer.");
670 const auto *NumBytes =
671 cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
672 Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
673
674 // Push live variables for the stack map (skipping the first two arguments
675 // <id> and <numBytes>).
676 if (!addStackMapLiveVars(Ops, I, 2))
677 return false;
678
679 // We are not adding any register mask info here, because the stackmap doesn't
680 // clobber anything.
681
682 // Add scratch registers as implicit def and early clobber.
683 CallingConv::ID CC = I->getCallingConv();
684 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
685 for (unsigned i = 0; ScratchRegs[i]; ++i)
687 ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
688 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
689
690 // Issue CALLSEQ_START
691 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
692 auto Builder =
693 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackDown));
694 const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
695 for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
696 Builder.addImm(0);
697
698 // Issue STACKMAP.
699 MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
700 TII.get(TargetOpcode::STACKMAP));
701 for (auto const &MO : Ops)
702 MIB.add(MO);
703
704 // Issue CALLSEQ_END
705 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
706 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackUp))
707 .addImm(0)
708 .addImm(0);
709
710 // Inform the Frame Information that we have a stackmap in this function.
712
713 return true;
714}
715
716/// Lower an argument list according to the target calling convention.
717///
718/// This is a helper for lowering intrinsics that follow a target calling
719/// convention or require stack pointer adjustment. Only a subset of the
720/// intrinsic's operands need to participate in the calling convention.
721bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
722 unsigned NumArgs, const Value *Callee,
723 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
724 ArgListTy Args;
725 Args.reserve(NumArgs);
726
727 // Populate the argument list.
728 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
729 Value *V = CI->getOperand(ArgI);
730
731 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
732
733 ArgListEntry Entry;
734 Entry.Val = V;
735 Entry.Ty = V->getType();
736 Entry.setAttributes(CI, ArgI);
737 Args.push_back(Entry);
738 }
739
740 Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
741 : CI->getType();
742 CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
743
744 return lowerCallTo(CLI);
745}
746
748 const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
749 StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
750 SmallString<32> MangledName;
751 Mangler::getNameWithPrefix(MangledName, Target, DL);
752 MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
753 return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
754}
755
757 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
758 // i32 <numBytes>,
759 // i8* <target>,
760 // i32 <numArgs>,
761 // [Args...],
762 // [live variables...])
763 CallingConv::ID CC = I->getCallingConv();
764 bool IsAnyRegCC = CC == CallingConv::AnyReg;
765 bool HasDef = !I->getType()->isVoidTy();
766 Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
767
768 // Get the real number of arguments participating in the call <numArgs>
769 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
770 "Expected a constant integer.");
771 const auto *NumArgsVal =
772 cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
773 unsigned NumArgs = NumArgsVal->getZExtValue();
774
775 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
776 // This includes all meta-operands up to but not including CC.
777 unsigned NumMetaOpers = PatchPointOpers::CCPos;
778 assert(I->arg_size() >= NumMetaOpers + NumArgs &&
779 "Not enough arguments provided to the patchpoint intrinsic");
780
781 // For AnyRegCC the arguments are lowered later on manually.
782 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
784 CLI.setIsPatchPoint();
785 if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
786 return false;
787
788 assert(CLI.Call && "No call instruction specified.");
789
791
792 // Add an explicit result reg if we use the anyreg calling convention.
793 if (IsAnyRegCC && HasDef) {
794 assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
796 CLI.NumResultRegs = 1;
797 Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true));
798 }
799
800 // Add the <id> and <numBytes> constants.
801 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
802 "Expected a constant integer.");
803 const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
804 Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
805
806 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
807 "Expected a constant integer.");
808 const auto *NumBytes =
809 cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
810 Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
811
812 // Add the call target.
813 if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
814 uint64_t CalleeConstAddr =
815 cast<ConstantInt>(C->getOperand(0))->getZExtValue();
816 Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
817 } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
818 if (C->getOpcode() == Instruction::IntToPtr) {
819 uint64_t CalleeConstAddr =
820 cast<ConstantInt>(C->getOperand(0))->getZExtValue();
821 Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
822 } else
823 llvm_unreachable("Unsupported ConstantExpr.");
824 } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
826 } else if (isa<ConstantPointerNull>(Callee))
828 else
829 llvm_unreachable("Unsupported callee address.");
830
831 // Adjust <numArgs> to account for any arguments that have been passed on
832 // the stack instead.
833 unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
834 Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
835
836 // Add the calling convention
837 Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
838
839 // Add the arguments we omitted previously. The register allocator should
840 // place these in any free register.
841 if (IsAnyRegCC) {
842 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
843 Register Reg = getRegForValue(I->getArgOperand(i));
844 if (!Reg)
845 return false;
846 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
847 }
848 }
849
850 // Push the arguments from the call instruction.
851 for (auto Reg : CLI.OutRegs)
852 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
853
854 // Push live variables for the stack map.
855 if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
856 return false;
857
858 // Push the register mask info.
861
862 // Add scratch registers as implicit def and early clobber.
863 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
864 for (unsigned i = 0; ScratchRegs[i]; ++i)
866 ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
867 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
868
869 // Add implicit defs (return values).
870 for (auto Reg : CLI.InRegs)
871 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/true,
872 /*isImp=*/true));
873
874 // Insert the patchpoint instruction before the call generated by the target.
876 TII.get(TargetOpcode::PATCHPOINT));
877
878 for (auto &MO : Ops)
879 MIB.add(MO);
880
882
883 // Delete the original call instruction.
884 CLI.Call->eraseFromParent();
885
886 // Inform the Frame Information that we have a patchpoint in this function.
887 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
888
889 if (CLI.NumResultRegs)
891 return true;
892}
893
895 const auto &Triple = TM.getTargetTriple();
897 return true; // don't do anything to this instruction.
900 /*isDef=*/false));
902 /*isDef=*/false));
904 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
905 TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
906 for (auto &MO : Ops)
907 MIB.add(MO);
908
909 // Insert the Patchable Event Call instruction, that gets lowered properly.
910 return true;
911}
912
914 const auto &Triple = TM.getTargetTriple();
916 return true; // don't do anything to this instruction.
919 /*isDef=*/false));
921 /*isDef=*/false));
923 /*isDef=*/false));
925 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
926 TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
927 for (auto &MO : Ops)
928 MIB.add(MO);
929
930 // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
931 return true;
932}
933
934/// Returns an AttributeList representing the attributes applied to the return
935/// value of the given call.
938 if (CLI.RetSExt)
939 Attrs.push_back(Attribute::SExt);
940 if (CLI.RetZExt)
941 Attrs.push_back(Attribute::ZExt);
942 if (CLI.IsInReg)
943 Attrs.push_back(Attribute::InReg);
944
946 Attrs);
947}
948
949bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
950 unsigned NumArgs) {
951 MCContext &Ctx = MF->getContext();
952 SmallString<32> MangledName;
953 Mangler::getNameWithPrefix(MangledName, SymName, DL);
954 MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
955 return lowerCallTo(CI, Sym, NumArgs);
956}
957
959 unsigned NumArgs) {
960 FunctionType *FTy = CI->getFunctionType();
961 Type *RetTy = CI->getType();
962
963 ArgListTy Args;
964 Args.reserve(NumArgs);
965
966 // Populate the argument list.
967 // Attributes for args start at offset 1, after the return attribute.
968 for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
969 Value *V = CI->getOperand(ArgI);
970
971 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
972
973 ArgListEntry Entry;
974 Entry.Val = V;
975 Entry.Ty = V->getType();
976 Entry.setAttributes(CI, ArgI);
977 Args.push_back(Entry);
978 }
980
982 CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), *CI, NumArgs);
983
984 return lowerCallTo(CLI);
985}
986
988 // Handle the incoming return values from the call.
989 CLI.clearIns();
990 SmallVector<EVT, 4> RetTys;
991 ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
992
994 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
995
996 bool CanLowerReturn = TLI.CanLowerReturn(
997 CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
998
999 // FIXME: sret demotion isn't supported yet - bail out.
1000 if (!CanLowerReturn)
1001 return false;
1002
1003 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1004 EVT VT = RetTys[I];
1005 MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1006 unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1007 for (unsigned i = 0; i != NumRegs; ++i) {
1008 ISD::InputArg MyFlags;
1009 MyFlags.VT = RegisterVT;
1010 MyFlags.ArgVT = VT;
1011 MyFlags.Used = CLI.IsReturnValueUsed;
1012 if (CLI.RetSExt)
1013 MyFlags.Flags.setSExt();
1014 if (CLI.RetZExt)
1015 MyFlags.Flags.setZExt();
1016 if (CLI.IsInReg)
1017 MyFlags.Flags.setInReg();
1018 CLI.Ins.push_back(MyFlags);
1019 }
1020 }
1021
1022 // Handle all of the outgoing arguments.
1023 CLI.clearOuts();
1024 for (auto &Arg : CLI.getArgs()) {
1025 Type *FinalType = Arg.Ty;
1026 if (Arg.IsByVal)
1027 FinalType = Arg.IndirectType;
1029 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
1030
1032 if (Arg.IsZExt)
1033 Flags.setZExt();
1034 if (Arg.IsSExt)
1035 Flags.setSExt();
1036 if (Arg.IsInReg)
1037 Flags.setInReg();
1038 if (Arg.IsSRet)
1039 Flags.setSRet();
1040 if (Arg.IsSwiftSelf)
1041 Flags.setSwiftSelf();
1042 if (Arg.IsSwiftAsync)
1043 Flags.setSwiftAsync();
1044 if (Arg.IsSwiftError)
1045 Flags.setSwiftError();
1046 if (Arg.IsCFGuardTarget)
1047 Flags.setCFGuardTarget();
1048 if (Arg.IsByVal)
1049 Flags.setByVal();
1050 if (Arg.IsInAlloca) {
1051 Flags.setInAlloca();
1052 // Set the byval flag for CCAssignFn callbacks that don't know about
1053 // inalloca. This way we can know how many bytes we should've allocated
1054 // and how many bytes a callee cleanup function will pop. If we port
1055 // inalloca to more targets, we'll have to add custom inalloca handling in
1056 // the various CC lowering callbacks.
1057 Flags.setByVal();
1058 }
1059 if (Arg.IsPreallocated) {
1060 Flags.setPreallocated();
1061 // Set the byval flag for CCAssignFn callbacks that don't know about
1062 // preallocated. This way we can know how many bytes we should've
1063 // allocated and how many bytes a callee cleanup function will pop. If we
1064 // port preallocated to more targets, we'll have to add custom
1065 // preallocated handling in the various CC lowering callbacks.
1066 Flags.setByVal();
1067 }
1068 MaybeAlign MemAlign = Arg.Alignment;
1069 if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1070 unsigned FrameSize = DL.getTypeAllocSize(Arg.IndirectType);
1071
1072 // For ByVal, alignment should come from FE. BE will guess if this info
1073 // is not there, but there are cases it cannot get right.
1074 if (!MemAlign)
1075 MemAlign = Align(TLI.getByValTypeAlignment(Arg.IndirectType, DL));
1076 Flags.setByValSize(FrameSize);
1077 } else if (!MemAlign) {
1078 MemAlign = DL.getABITypeAlign(Arg.Ty);
1079 }
1080 Flags.setMemAlign(*MemAlign);
1081 if (Arg.IsNest)
1082 Flags.setNest();
1083 if (NeedsRegBlock)
1084 Flags.setInConsecutiveRegs();
1085 Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
1086 CLI.OutVals.push_back(Arg.Val);
1087 CLI.OutFlags.push_back(Flags);
1088 }
1089
1090 if (!fastLowerCall(CLI))
1091 return false;
1092
1093 // Set all unused physreg defs as dead.
1094 assert(CLI.Call && "No call instruction specified.");
1096
1097 if (CLI.NumResultRegs && CLI.CB)
1099
1100 // Set labels for heapallocsite call.
1101 if (CLI.CB)
1102 if (MDNode *MD = CLI.CB->getMetadata("heapallocsite"))
1103 CLI.Call->setHeapAllocMarker(*MF, MD);
1104
1105 return true;
1106}
1107
1109 FunctionType *FuncTy = CI->getFunctionType();
1110 Type *RetTy = CI->getType();
1111
1112 ArgListTy Args;
1113 ArgListEntry Entry;
1114 Args.reserve(CI->arg_size());
1115
1116 for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1117 Value *V = *i;
1118
1119 // Skip empty types
1120 if (V->getType()->isEmptyTy())
1121 continue;
1122
1123 Entry.Val = V;
1124 Entry.Ty = V->getType();
1125
1126 // Skip the first return-type Attribute to get to params.
1127 Entry.setAttributes(CI, i - CI->arg_begin());
1128 Args.push_back(Entry);
1129 }
1130
1131 // Check if target-independent constraints permit a tail call here.
1132 // Target-dependent constraints are checked within fastLowerCall.
1133 bool IsTailCall = CI->isTailCall();
1134 if (IsTailCall && !isInTailCallPosition(*CI, TM))
1135 IsTailCall = false;
1136 if (IsTailCall && !CI->isMustTailCall() &&
1137 MF->getFunction().getFnAttribute("disable-tail-calls").getValueAsBool())
1138 IsTailCall = false;
1139
1140 CallLoweringInfo CLI;
1141 CLI.setCallee(RetTy, FuncTy, CI->getCalledOperand(), std::move(Args), *CI)
1142 .setTailCall(IsTailCall);
1143
1144 diagnoseDontCall(*CI);
1145
1146 return lowerCallTo(CLI);
1147}
1148
1150 const CallInst *Call = cast<CallInst>(I);
1151
1152 // Handle simple inline asms.
1153 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledOperand())) {
1154 // Don't attempt to handle constraints.
1155 if (!IA->getConstraintString().empty())
1156 return false;
1157
1158 unsigned ExtraInfo = 0;
1159 if (IA->hasSideEffects())
1161 if (IA->isAlignStack())
1162 ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1163 if (Call->isConvergent())
1164 ExtraInfo |= InlineAsm::Extra_IsConvergent;
1165 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1166
1167 MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1168 TII.get(TargetOpcode::INLINEASM));
1169 MIB.addExternalSymbol(IA->getAsmString().c_str());
1170 MIB.addImm(ExtraInfo);
1171
1172 const MDNode *SrcLoc = Call->getMetadata("srcloc");
1173 if (SrcLoc)
1174 MIB.addMetadata(SrcLoc);
1175
1176 return true;
1177 }
1178
1179 // Handle intrinsic function calls.
1180 if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1181 return selectIntrinsicCall(II);
1182
1183 return lowerCall(Call);
1184}
1185
1187 switch (II->getIntrinsicID()) {
1188 default:
1189 break;
1190 // At -O0 we don't care about the lifetime intrinsics.
1191 case Intrinsic::lifetime_start:
1192 case Intrinsic::lifetime_end:
1193 // The donothing intrinsic does, well, nothing.
1194 case Intrinsic::donothing:
1195 // Neither does the sideeffect intrinsic.
1196 case Intrinsic::sideeffect:
1197 // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1198 case Intrinsic::assume:
1199 // Neither does the llvm.experimental.noalias.scope.decl intrinsic
1200 case Intrinsic::experimental_noalias_scope_decl:
1201 return true;
1202 case Intrinsic::dbg_declare: {
1203 const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1204 assert(DI->getVariable() && "Missing variable");
1205 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1206 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
1207 << " (!hasDebugInfo)\n");
1208 return true;
1209 }
1210
1211 const Value *Address = DI->getAddress();
1212 if (!Address || isa<UndefValue>(Address)) {
1213 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
1214 << " (bad/undef address)\n");
1215 return true;
1216 }
1217
1218 // Byval arguments with frame indices were already handled after argument
1219 // lowering and before isel.
1220 const auto *Arg =
1221 dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
1222 if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1223 return true;
1224
1225 std::optional<MachineOperand> Op;
1227 Op = MachineOperand::CreateReg(Reg, false);
1228
1229 // If we have a VLA that has a "use" in a metadata node that's then used
1230 // here but it has no other uses, then we have a problem. E.g.,
1231 //
1232 // int foo (const int *x) {
1233 // char a[*x];
1234 // return 0;
1235 // }
1236 //
1237 // If we assign 'a' a vreg and fast isel later on has to use the selection
1238 // DAG isel, it will want to copy the value to the vreg. However, there are
1239 // no uses, which goes counter to what selection DAG isel expects.
1240 if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1241 (!isa<AllocaInst>(Address) ||
1242 !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1243 Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1244 false);
1245
1246 if (Op) {
1248 "Expected inlined-at fields to agree");
1249 if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1250 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1251 // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1252 // the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1254 {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1256 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(),
1257 TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, *Op,
1258 DI->getVariable(), NewExpr);
1259 } else {
1260 // A dbg.declare describes the address of a source variable, so lower it
1261 // into an indirect DBG_VALUE.
1262 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(),
1263 TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, *Op,
1264 DI->getVariable(), DI->getExpression());
1265 }
1266 } else {
1267 // We can't yet handle anything else here because it would require
1268 // generating code, thus altering codegen because of debug info.
1269 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
1270 << " (no materialized reg for address)\n");
1271 }
1272 return true;
1273 }
1274 case Intrinsic::dbg_value: {
1275 // This form of DBG_VALUE is target-independent.
1276 const DbgValueInst *DI = cast<DbgValueInst>(II);
1277 const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1278 const Value *V = DI->getValue();
1280 "Expected inlined-at fields to agree");
1281 if (!V || isa<UndefValue>(V) || DI->hasArgList()) {
1282 // DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1283 // undef DBG_VALUE to terminate any prior location.
1284 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, false, 0U,
1285 DI->getVariable(), DI->getExpression());
1286 } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1287 // See if there's an expression to constant-fold.
1288 DIExpression *Expr = DI->getExpression();
1289 if (Expr)
1290 std::tie(Expr, CI) = Expr->constantFold(CI);
1291 if (CI->getBitWidth() > 64)
1292 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
1293 .addCImm(CI)
1294 .addImm(0U)
1295 .addMetadata(DI->getVariable())
1296 .addMetadata(Expr);
1297 else
1298 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
1299 .addImm(CI->getZExtValue())
1300 .addImm(0U)
1301 .addMetadata(DI->getVariable())
1302 .addMetadata(Expr);
1303 } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1304 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
1305 .addFPImm(CF)
1306 .addImm(0U)
1307 .addMetadata(DI->getVariable())
1308 .addMetadata(DI->getExpression());
1309 } else if (Register Reg = lookUpRegForValue(V)) {
1310 // FIXME: This does not handle register-indirect values at offset 0.
1311 if (!FuncInfo.MF->useDebugInstrRef()) {
1312 bool IsIndirect = false;
1313 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, IsIndirect,
1314 Reg, DI->getVariable(), DI->getExpression());
1315 } else {
1316 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1317 // to be later patched up by finalizeDebugInstrRefs.
1319 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1320 /* isKill */ false, /* isDead */ false,
1321 /* isUndef */ false, /* isEarlyClobber */ false,
1322 /* SubReg */ 0, /* isDebug */ true)});
1325 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(),
1326 TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1327 DI->getVariable(), NewExpr);
1328 }
1329 } else {
1330 // We don't know how to handle other cases, so we drop.
1331 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1332 }
1333 return true;
1334 }
1335 case Intrinsic::dbg_label: {
1336 const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1337 assert(DI->getLabel() && "Missing label");
1338 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1339 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1340 return true;
1341 }
1342
1343 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1344 TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1345 return true;
1346 }
1347 case Intrinsic::objectsize:
1348 llvm_unreachable("llvm.objectsize.* should have been lowered already");
1349
1350 case Intrinsic::is_constant:
1351 llvm_unreachable("llvm.is.constant.* should have been lowered already");
1352
1353 case Intrinsic::launder_invariant_group:
1354 case Intrinsic::strip_invariant_group:
1355 case Intrinsic::expect: {
1356 Register ResultReg = getRegForValue(II->getArgOperand(0));
1357 if (!ResultReg)
1358 return false;
1359 updateValueMap(II, ResultReg);
1360 return true;
1361 }
1362 case Intrinsic::experimental_stackmap:
1363 return selectStackmap(II);
1364 case Intrinsic::experimental_patchpoint_void:
1365 case Intrinsic::experimental_patchpoint_i64:
1366 return selectPatchpoint(II);
1367
1368 case Intrinsic::xray_customevent:
1369 return selectXRayCustomEvent(II);
1370 case Intrinsic::xray_typedevent:
1371 return selectXRayTypedEvent(II);
1372 }
1373
1374 return fastLowerIntrinsicCall(II);
1375}
1376
1377bool FastISel::selectCast(const User *I, unsigned Opcode) {
1378 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1379 EVT DstVT = TLI.getValueType(DL, I->getType());
1380
1381 if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1382 !DstVT.isSimple())
1383 // Unhandled type. Halt "fast" selection and bail.
1384 return false;
1385
1386 // Check if the destination type is legal.
1387 if (!TLI.isTypeLegal(DstVT))
1388 return false;
1389
1390 // Check if the source operand is legal.
1391 if (!TLI.isTypeLegal(SrcVT))
1392 return false;
1393
1394 Register InputReg = getRegForValue(I->getOperand(0));
1395 if (!InputReg)
1396 // Unhandled operand. Halt "fast" selection and bail.
1397 return false;
1398
1399 Register ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1400 Opcode, InputReg);
1401 if (!ResultReg)
1402 return false;
1403
1404 updateValueMap(I, ResultReg);
1405 return true;
1406}
1407
1409 EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1410 EVT DstEVT = TLI.getValueType(DL, I->getType());
1411 if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1412 !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1413 // Unhandled type. Halt "fast" selection and bail.
1414 return false;
1415
1416 MVT SrcVT = SrcEVT.getSimpleVT();
1417 MVT DstVT = DstEVT.getSimpleVT();
1418 Register Op0 = getRegForValue(I->getOperand(0));
1419 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1420 return false;
1421
1422 // If the bitcast doesn't change the type, just use the operand value.
1423 if (SrcVT == DstVT) {
1424 updateValueMap(I, Op0);
1425 return true;
1426 }
1427
1428 // Otherwise, select a BITCAST opcode.
1429 Register ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0);
1430 if (!ResultReg)
1431 return false;
1432
1433 updateValueMap(I, ResultReg);
1434 return true;
1435}
1436
1438 Register Reg = getRegForValue(I->getOperand(0));
1439 if (!Reg)
1440 // Unhandled operand.
1441 return false;
1442
1443 EVT ETy = TLI.getValueType(DL, I->getOperand(0)->getType());
1444 if (ETy == MVT::Other || !TLI.isTypeLegal(ETy))
1445 // Unhandled type, bail out.
1446 return false;
1447
1448 MVT Ty = ETy.getSimpleVT();
1449 const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(Ty);
1450 Register ResultReg = createResultReg(TyRegClass);
1451 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1452 TII.get(TargetOpcode::COPY), ResultReg).addReg(Reg);
1453
1454 updateValueMap(I, ResultReg);
1455 return true;
1456}
1457
1458// Remove local value instructions starting from the instruction after
1459// SavedLastLocalValue to the current function insert point.
1460void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1461{
1462 MachineInstr *CurLastLocalValue = getLastLocalValue();
1463 if (CurLastLocalValue != SavedLastLocalValue) {
1464 // Find the first local value instruction to be deleted.
1465 // This is the instruction after SavedLastLocalValue if it is non-NULL.
1466 // Otherwise it's the first instruction in the block.
1467 MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1468 if (SavedLastLocalValue)
1469 ++FirstDeadInst;
1470 else
1471 FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1472 setLastLocalValue(SavedLastLocalValue);
1473 removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1474 }
1475}
1476
1478 // Flush the local value map before starting each instruction.
1479 // This improves locality and debugging, and can reduce spills.
1480 // Reuse of values across IR instructions is relatively uncommon.
1481 flushLocalValueMap();
1482
1483 MachineInstr *SavedLastLocalValue = getLastLocalValue();
1484 // Just before the terminator instruction, insert instructions to
1485 // feed PHI nodes in successor blocks.
1486 if (I->isTerminator()) {
1487 if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1488 // PHI node handling may have generated local value instructions,
1489 // even though it failed to handle all PHI nodes.
1490 // We remove these instructions because SelectionDAGISel will generate
1491 // them again.
1492 removeDeadLocalValueCode(SavedLastLocalValue);
1493 return false;
1494 }
1495 }
1496
1497 // FastISel does not handle any operand bundles except OB_funclet.
1498 if (auto *Call = dyn_cast<CallBase>(I))
1499 for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1500 if (Call->getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1501 return false;
1502
1503 MIMD = MIMetadata(*I);
1504
1505 SavedInsertPt = FuncInfo.InsertPt;
1506
1507 if (const auto *Call = dyn_cast<CallInst>(I)) {
1508 const Function *F = Call->getCalledFunction();
1509 LibFunc Func;
1510
1511 // As a special case, don't handle calls to builtin library functions that
1512 // may be translated directly to target instructions.
1513 if (F && !F->hasLocalLinkage() && F->hasName() &&
1514 LibInfo->getLibFunc(F->getName(), Func) &&
1516 return false;
1517
1518 // Don't handle Intrinsic::trap if a trap function is specified.
1519 if (F && F->getIntrinsicID() == Intrinsic::trap &&
1520 Call->hasFnAttr("trap-func-name"))
1521 return false;
1522 }
1523
1524 // First, try doing target-independent selection.
1526 if (selectOperator(I, I->getOpcode())) {
1527 ++NumFastIselSuccessIndependent;
1528 MIMD = {};
1529 return true;
1530 }
1531 // Remove dead code.
1533 if (SavedInsertPt != FuncInfo.InsertPt)
1534 removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1535 SavedInsertPt = FuncInfo.InsertPt;
1536 }
1537 // Next, try calling the target to attempt to handle the instruction.
1538 if (fastSelectInstruction(I)) {
1539 ++NumFastIselSuccessTarget;
1540 MIMD = {};
1541 return true;
1542 }
1543 // Remove dead code.
1545 if (SavedInsertPt != FuncInfo.InsertPt)
1546 removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1547
1548 MIMD = {};
1549 // Undo phi node updates, because they will be added again by SelectionDAG.
1550 if (I->isTerminator()) {
1551 // PHI node handling may have generated local value instructions.
1552 // We remove them because SelectionDAGISel will generate them again.
1553 removeDeadLocalValueCode(SavedLastLocalValue);
1554 FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1555 }
1556 return false;
1557}
1558
1559/// Emit an unconditional branch to the given block, unless it is the immediate
1560/// (fall-through) successor, and update the CFG.
1562 const DebugLoc &DbgLoc) {
1563 if (FuncInfo.MBB->getBasicBlock()->sizeWithoutDebug() > 1 &&
1564 FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1565 // For more accurate line information if this is the only non-debug
1566 // instruction in the block then emit it, otherwise we have the
1567 // unconditional fall-through case, which needs no instructions.
1568 } else {
1569 // The unconditional branch case.
1570 TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1572 }
1573 if (FuncInfo.BPI) {
1574 auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1575 FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1576 FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1577 } else
1578 FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1579}
1580
1582 MachineBasicBlock *TrueMBB,
1583 MachineBasicBlock *FalseMBB) {
1584 // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1585 // happen in degenerate IR and MachineIR forbids to have a block twice in the
1586 // successor/predecessor lists.
1587 if (TrueMBB != FalseMBB) {
1588 if (FuncInfo.BPI) {
1589 auto BranchProbability =
1590 FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1591 FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1592 } else
1593 FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1594 }
1595
1596 fastEmitBranch(FalseMBB, MIMD.getDL());
1597}
1598
1599/// Emit an FNeg operation.
1600bool FastISel::selectFNeg(const User *I, const Value *In) {
1601 Register OpReg = getRegForValue(In);
1602 if (!OpReg)
1603 return false;
1604
1605 // If the target has ISD::FNEG, use it.
1606 EVT VT = TLI.getValueType(DL, I->getType());
1607 Register ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1608 OpReg);
1609 if (ResultReg) {
1610 updateValueMap(I, ResultReg);
1611 return true;
1612 }
1613
1614 // Bitcast the value to integer, twiddle the sign bit with xor,
1615 // and then bitcast it back to floating-point.
1616 if (VT.getSizeInBits() > 64)
1617 return false;
1618 EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1619 if (!TLI.isTypeLegal(IntVT))
1620 return false;
1621
1622 Register IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1623 ISD::BITCAST, OpReg);
1624 if (!IntReg)
1625 return false;
1626
1627 Register IntResultReg = fastEmit_ri_(
1628 IntVT.getSimpleVT(), ISD::XOR, IntReg,
1629 UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1630 if (!IntResultReg)
1631 return false;
1632
1633 ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1634 IntResultReg);
1635 if (!ResultReg)
1636 return false;
1637
1638 updateValueMap(I, ResultReg);
1639 return true;
1640}
1641
1643 const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1644 if (!EVI)
1645 return false;
1646
1647 // Make sure we only try to handle extracts with a legal result. But also
1648 // allow i1 because it's easy.
1649 EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1650 if (!RealVT.isSimple())
1651 return false;
1652 MVT VT = RealVT.getSimpleVT();
1653 if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1654 return false;
1655
1656 const Value *Op0 = EVI->getOperand(0);
1657 Type *AggTy = Op0->getType();
1658
1659 // Get the base result register.
1660 unsigned ResultReg;
1662 if (I != FuncInfo.ValueMap.end())
1663 ResultReg = I->second;
1664 else if (isa<Instruction>(Op0))
1665 ResultReg = FuncInfo.InitializeRegForValue(Op0);
1666 else
1667 return false; // fast-isel can't handle aggregate constants at the moment
1668
1669 // Get the actual result register, which is an offset from the base register.
1670 unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1671
1672 SmallVector<EVT, 4> AggValueVTs;
1673 ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1674
1675 for (unsigned i = 0; i < VTIndex; i++)
1676 ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1677
1678 updateValueMap(EVI, ResultReg);
1679 return true;
1680}
1681
1682bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1683 switch (Opcode) {
1684 case Instruction::Add:
1685 return selectBinaryOp(I, ISD::ADD);
1686 case Instruction::FAdd:
1687 return selectBinaryOp(I, ISD::FADD);
1688 case Instruction::Sub:
1689 return selectBinaryOp(I, ISD::SUB);
1690 case Instruction::FSub:
1691 return selectBinaryOp(I, ISD::FSUB);
1692 case Instruction::Mul:
1693 return selectBinaryOp(I, ISD::MUL);
1694 case Instruction::FMul:
1695 return selectBinaryOp(I, ISD::FMUL);
1696 case Instruction::SDiv:
1697 return selectBinaryOp(I, ISD::SDIV);
1698 case Instruction::UDiv:
1699 return selectBinaryOp(I, ISD::UDIV);
1700 case Instruction::FDiv:
1701 return selectBinaryOp(I, ISD::FDIV);
1702 case Instruction::SRem:
1703 return selectBinaryOp(I, ISD::SREM);
1704 case Instruction::URem:
1705 return selectBinaryOp(I, ISD::UREM);
1706 case Instruction::FRem:
1707 return selectBinaryOp(I, ISD::FREM);
1708 case Instruction::Shl:
1709 return selectBinaryOp(I, ISD::SHL);
1710 case Instruction::LShr:
1711 return selectBinaryOp(I, ISD::SRL);
1712 case Instruction::AShr:
1713 return selectBinaryOp(I, ISD::SRA);
1714 case Instruction::And:
1715 return selectBinaryOp(I, ISD::AND);
1716 case Instruction::Or:
1717 return selectBinaryOp(I, ISD::OR);
1718 case Instruction::Xor:
1719 return selectBinaryOp(I, ISD::XOR);
1720
1721 case Instruction::FNeg:
1722 return selectFNeg(I, I->getOperand(0));
1723
1724 case Instruction::GetElementPtr:
1725 return selectGetElementPtr(I);
1726
1727 case Instruction::Br: {
1728 const BranchInst *BI = cast<BranchInst>(I);
1729
1730 if (BI->isUnconditional()) {
1731 const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1732 MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1733 fastEmitBranch(MSucc, BI->getDebugLoc());
1734 return true;
1735 }
1736
1737 // Conditional branches are not handed yet.
1738 // Halt "fast" selection and bail.
1739 return false;
1740 }
1741
1742 case Instruction::Unreachable:
1745 else
1746 return true;
1747
1748 case Instruction::Alloca:
1749 // FunctionLowering has the static-sized case covered.
1750 if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1751 return true;
1752
1753 // Dynamic-sized alloca is not handled yet.
1754 return false;
1755
1756 case Instruction::Call:
1757 // On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1758 // callee of the direct function call instruction will be mapped to the
1759 // symbol for the function's entry point, which is distinct from the
1760 // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1761 // name is the C-linkage name of the source level function.
1762 // But fast isel still has the ability to do selection for intrinsics.
1763 if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(I))
1764 return false;
1765 return selectCall(I);
1766
1767 case Instruction::BitCast:
1768 return selectBitCast(I);
1769
1770 case Instruction::FPToSI:
1771 return selectCast(I, ISD::FP_TO_SINT);
1772 case Instruction::ZExt:
1773 return selectCast(I, ISD::ZERO_EXTEND);
1774 case Instruction::SExt:
1775 return selectCast(I, ISD::SIGN_EXTEND);
1776 case Instruction::Trunc:
1777 return selectCast(I, ISD::TRUNCATE);
1778 case Instruction::SIToFP:
1779 return selectCast(I, ISD::SINT_TO_FP);
1780
1781 case Instruction::IntToPtr: // Deliberate fall-through.
1782 case Instruction::PtrToInt: {
1783 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1784 EVT DstVT = TLI.getValueType(DL, I->getType());
1785 if (DstVT.bitsGT(SrcVT))
1786 return selectCast(I, ISD::ZERO_EXTEND);
1787 if (DstVT.bitsLT(SrcVT))
1788 return selectCast(I, ISD::TRUNCATE);
1789 Register Reg = getRegForValue(I->getOperand(0));
1790 if (!Reg)
1791 return false;
1792 updateValueMap(I, Reg);
1793 return true;
1794 }
1795
1796 case Instruction::ExtractValue:
1797 return selectExtractValue(I);
1798
1799 case Instruction::Freeze:
1800 return selectFreeze(I);
1801
1802 case Instruction::PHI:
1803 llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1804
1805 default:
1806 // Unhandled instruction. Halt "fast" selection and bail.
1807 return false;
1808 }
1809}
1810
1814 : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1815 MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1816 TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1817 TII(*MF->getSubtarget().getInstrInfo()),
1818 TLI(*MF->getSubtarget().getTargetLowering()),
1819 TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1821
1822FastISel::~FastISel() = default;
1823
1824bool FastISel::fastLowerArguments() { return false; }
1825
1826bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1827
1829 return false;
1830}
1831
1832unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1833
1834unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/) {
1835 return 0;
1836}
1837
1838unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1839 unsigned /*Op1*/) {
1840 return 0;
1841}
1842
1843unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1844 return 0;
1845}
1846
1847unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1848 const ConstantFP * /*FPImm*/) {
1849 return 0;
1850}
1851
1852unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1853 uint64_t /*Imm*/) {
1854 return 0;
1855}
1856
1857/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1858/// instruction with an immediate operand using fastEmit_ri.
1859/// If that fails, it materializes the immediate into a register and try
1860/// fastEmit_rr instead.
1861Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1862 uint64_t Imm, MVT ImmType) {
1863 // If this is a multiply by a power of two, emit this as a shift left.
1864 if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1865 Opcode = ISD::SHL;
1866 Imm = Log2_64(Imm);
1867 } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1868 // div x, 8 -> srl x, 3
1869 Opcode = ISD::SRL;
1870 Imm = Log2_64(Imm);
1871 }
1872
1873 // Horrible hack (to be removed), check to make sure shift amounts are
1874 // in-range.
1875 if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1876 Imm >= VT.getSizeInBits())
1877 return 0;
1878
1879 // First check if immediate type is legal. If not, we can't use the ri form.
1880 Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
1881 if (ResultReg)
1882 return ResultReg;
1883 Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1884 if (!MaterialReg) {
1885 // This is a bit ugly/slow, but failing here means falling out of
1886 // fast-isel, which would be very slow.
1887 IntegerType *ITy =
1888 IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits());
1889 MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
1890 if (!MaterialReg)
1891 return 0;
1892 }
1893 return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
1894}
1895
1897 return MRI.createVirtualRegister(RC);
1898}
1899
1901 unsigned OpNum) {
1902 if (Op.isVirtual()) {
1903 const TargetRegisterClass *RegClass =
1904 TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
1905 if (!MRI.constrainRegClass(Op, RegClass)) {
1906 // If it's not legal to COPY between the register classes, something
1907 // has gone very wrong before we got here.
1908 Register NewOp = createResultReg(RegClass);
1909 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1910 TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
1911 return NewOp;
1912 }
1913 }
1914 return Op;
1915}
1916
1917Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
1918 const TargetRegisterClass *RC) {
1919 Register ResultReg = createResultReg(RC);
1920 const MCInstrDesc &II = TII.get(MachineInstOpcode);
1921
1922 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg);
1923 return ResultReg;
1924}
1925
1926Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
1927 const TargetRegisterClass *RC, unsigned Op0) {
1928 const MCInstrDesc &II = TII.get(MachineInstOpcode);
1929
1930 Register ResultReg = createResultReg(RC);
1931 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
1932
1933 if (II.getNumDefs() >= 1)
1934 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1935 .addReg(Op0);
1936 else {
1937 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
1938 .addReg(Op0);
1939 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
1940 ResultReg)
1941 .addReg(II.implicit_defs()[0]);
1942 }
1943
1944 return ResultReg;
1945}
1946
1947Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
1948 const TargetRegisterClass *RC, unsigned Op0,
1949 unsigned Op1) {
1950 const MCInstrDesc &II = TII.get(MachineInstOpcode);
1951
1952 Register ResultReg = createResultReg(RC);
1953 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
1954 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
1955
1956 if (II.getNumDefs() >= 1)
1957 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1958 .addReg(Op0)
1959 .addReg(Op1);
1960 else {
1961 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
1962 .addReg(Op0)
1963 .addReg(Op1);
1964 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
1965 ResultReg)
1966 .addReg(II.implicit_defs()[0]);
1967 }
1968 return ResultReg;
1969}
1970
1971Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
1972 const TargetRegisterClass *RC, unsigned Op0,
1973 unsigned Op1, unsigned Op2) {
1974 const MCInstrDesc &II = TII.get(MachineInstOpcode);
1975
1976 Register ResultReg = createResultReg(RC);
1977 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
1978 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
1979 Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
1980
1981 if (II.getNumDefs() >= 1)
1982 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
1983 .addReg(Op0)
1984 .addReg(Op1)
1985 .addReg(Op2);
1986 else {
1987 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
1988 .addReg(Op0)
1989 .addReg(Op1)
1990 .addReg(Op2);
1991 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
1992 ResultReg)
1993 .addReg(II.implicit_defs()[0]);
1994 }
1995 return ResultReg;
1996}
1997
1998Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
1999 const TargetRegisterClass *RC, unsigned Op0,
2000 uint64_t Imm) {
2001 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2002
2003 Register ResultReg = createResultReg(RC);
2004 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2005
2006 if (II.getNumDefs() >= 1)
2007 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2008 .addReg(Op0)
2009 .addImm(Imm);
2010 else {
2011 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2012 .addReg(Op0)
2013 .addImm(Imm);
2014 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2015 ResultReg)
2016 .addReg(II.implicit_defs()[0]);
2017 }
2018 return ResultReg;
2019}
2020
2021Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2022 const TargetRegisterClass *RC, unsigned Op0,
2023 uint64_t Imm1, uint64_t Imm2) {
2024 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2025
2026 Register ResultReg = createResultReg(RC);
2027 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2028
2029 if (II.getNumDefs() >= 1)
2030 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2031 .addReg(Op0)
2032 .addImm(Imm1)
2033 .addImm(Imm2);
2034 else {
2035 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2036 .addReg(Op0)
2037 .addImm(Imm1)
2038 .addImm(Imm2);
2039 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2040 ResultReg)
2041 .addReg(II.implicit_defs()[0]);
2042 }
2043 return ResultReg;
2044}
2045
2046Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2047 const TargetRegisterClass *RC,
2048 const ConstantFP *FPImm) {
2049 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2050
2051 Register ResultReg = createResultReg(RC);
2052
2053 if (II.getNumDefs() >= 1)
2054 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2055 .addFPImm(FPImm);
2056 else {
2057 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2058 .addFPImm(FPImm);
2059 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2060 ResultReg)
2061 .addReg(II.implicit_defs()[0]);
2062 }
2063 return ResultReg;
2064}
2065
2066Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2067 const TargetRegisterClass *RC, unsigned Op0,
2068 unsigned Op1, uint64_t Imm) {
2069 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2070
2071 Register ResultReg = createResultReg(RC);
2072 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2073 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2074
2075 if (II.getNumDefs() >= 1)
2076 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2077 .addReg(Op0)
2078 .addReg(Op1)
2079 .addImm(Imm);
2080 else {
2081 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2082 .addReg(Op0)
2083 .addReg(Op1)
2084 .addImm(Imm);
2085 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2086 ResultReg)
2087 .addReg(II.implicit_defs()[0]);
2088 }
2089 return ResultReg;
2090}
2091
2092Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2093 const TargetRegisterClass *RC, uint64_t Imm) {
2094 Register ResultReg = createResultReg(RC);
2095 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2096
2097 if (II.getNumDefs() >= 1)
2098 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2099 .addImm(Imm);
2100 else {
2101 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).addImm(Imm);
2102 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2103 ResultReg)
2104 .addReg(II.implicit_defs()[0]);
2105 }
2106 return ResultReg;
2107}
2108
2110 uint32_t Idx) {
2111 Register ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2113 "Cannot yet extract from physregs");
2114 const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2116 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2117 ResultReg).addReg(Op0, 0, Idx);
2118 return ResultReg;
2119}
2120
2121/// Emit MachineInstrs to compute the value of Op with all but the least
2122/// significant bit set to zero.
2124 return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2125}
2126
2127/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2128/// Emit code to ensure constants are copied into registers when needed.
2129/// Remember the virtual registers that need to be added to the Machine PHI
2130/// nodes as input. We cannot just directly add them, because expansion
2131/// might result in multiple MBB's for one BB. As such, the start of the
2132/// BB might correspond to a different MBB than the end.
2133bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2134 const Instruction *TI = LLVMBB->getTerminator();
2135
2137 FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2138
2139 // Check successor nodes' PHI nodes that expect a constant to be available
2140 // from this block.
2141 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2142 const BasicBlock *SuccBB = TI->getSuccessor(succ);
2143 if (!isa<PHINode>(SuccBB->begin()))
2144 continue;
2145 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2146
2147 // If this terminator has multiple identical successors (common for
2148 // switches), only handle each succ once.
2149 if (!SuccsHandled.insert(SuccMBB).second)
2150 continue;
2151
2153
2154 // At this point we know that there is a 1-1 correspondence between LLVM PHI
2155 // nodes and Machine PHI nodes, but the incoming operands have not been
2156 // emitted yet.
2157 for (const PHINode &PN : SuccBB->phis()) {
2158 // Ignore dead phi's.
2159 if (PN.use_empty())
2160 continue;
2161
2162 // Only handle legal types. Two interesting things to note here. First,
2163 // by bailing out early, we may leave behind some dead instructions,
2164 // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2165 // own moves. Second, this check is necessary because FastISel doesn't
2166 // use CreateRegs to create registers, so it always creates
2167 // exactly one register for each non-void instruction.
2168 EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2169 if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2170 // Handle integer promotions, though, because they're common and easy.
2171 if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2172 FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
2173 return false;
2174 }
2175 }
2176
2177 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2178
2179 // Set the DebugLoc for the copy. Use the location of the operand if
2180 // there is one; otherwise no location, flushLocalValueMap will fix it.
2181 MIMD = {};
2182 if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2183 MIMD = MIMetadata(*Inst);
2184
2185 Register Reg = getRegForValue(PHIOp);
2186 if (!Reg) {
2187 FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
2188 return false;
2189 }
2190 FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2191 MIMD = {};
2192 }
2193 }
2194
2195 return true;
2196}
2197
2198bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2199 assert(LI->hasOneUse() &&
2200 "tryToFoldLoad expected a LoadInst with a single use");
2201 // We know that the load has a single use, but don't know what it is. If it
2202 // isn't one of the folded instructions, then we can't succeed here. Handle
2203 // this by scanning the single-use users of the load until we get to FoldInst.
2204 unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2205
2206 const Instruction *TheUser = LI->user_back();
2207 while (TheUser != FoldInst && // Scan up until we find FoldInst.
2208 // Stay in the right block.
2209 TheUser->getParent() == FoldInst->getParent() &&
2210 --MaxUsers) { // Don't scan too far.
2211 // If there are multiple or no uses of this instruction, then bail out.
2212 if (!TheUser->hasOneUse())
2213 return false;
2214
2215 TheUser = TheUser->user_back();
2216 }
2217
2218 // If we didn't find the fold instruction, then we failed to collapse the
2219 // sequence.
2220 if (TheUser != FoldInst)
2221 return false;
2222
2223 // Don't try to fold volatile loads. Target has to deal with alignment
2224 // constraints.
2225 if (LI->isVolatile())
2226 return false;
2227
2228 // Figure out which vreg this is going into. If there is no assigned vreg yet
2229 // then there actually was no reference to it. Perhaps the load is referenced
2230 // by a dead instruction.
2231 Register LoadReg = getRegForValue(LI);
2232 if (!LoadReg)
2233 return false;
2234
2235 // We can't fold if this vreg has no uses or more than one use. Multiple uses
2236 // may mean that the instruction got lowered to multiple MIs, or the use of
2237 // the loaded value ended up being multiple operands of the result.
2238 if (!MRI.hasOneUse(LoadReg))
2239 return false;
2240
2241 // If the register has fixups, there may be additional uses through a
2242 // different alias of the register.
2243 if (FuncInfo.RegsWithFixups.contains(LoadReg))
2244 return false;
2245
2247 MachineInstr *User = RI->getParent();
2248
2249 // Set the insertion point properly. Folding the load can cause generation of
2250 // other random instructions (like sign extends) for addressing modes; make
2251 // sure they get inserted in a logical place before the new instruction.
2252 FuncInfo.InsertPt = User;
2253 FuncInfo.MBB = User->getParent();
2254
2255 // Ask the target to try folding the load.
2256 return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2257}
2258
2260 // Must be an add.
2261 if (!isa<AddOperator>(Add))
2262 return false;
2263 // Type size needs to match.
2264 if (DL.getTypeSizeInBits(GEP->getType()) !=
2265 DL.getTypeSizeInBits(Add->getType()))
2266 return false;
2267 // Must be in the same basic block.
2268 if (isa<Instruction>(Add) &&
2269 FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2270 return false;
2271 // Must have a constant operand.
2272 return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2273}
2274
2277 const Value *Ptr;
2278 Type *ValTy;
2279 MaybeAlign Alignment;
2281 bool IsVolatile;
2282
2283 if (const auto *LI = dyn_cast<LoadInst>(I)) {
2284 Alignment = LI->getAlign();
2285 IsVolatile = LI->isVolatile();
2287 Ptr = LI->getPointerOperand();
2288 ValTy = LI->getType();
2289 } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2290 Alignment = SI->getAlign();
2291 IsVolatile = SI->isVolatile();
2293 Ptr = SI->getPointerOperand();
2294 ValTy = SI->getValueOperand()->getType();
2295 } else
2296 return nullptr;
2297
2298 bool IsNonTemporal = I->hasMetadata(LLVMContext::MD_nontemporal);
2299 bool IsInvariant = I->hasMetadata(LLVMContext::MD_invariant_load);
2300 bool IsDereferenceable = I->hasMetadata(LLVMContext::MD_dereferenceable);
2301 const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2302
2303 AAMDNodes AAInfo = I->getAAMetadata();
2304
2305 if (!Alignment) // Ensure that codegen never sees alignment 0.
2306 Alignment = DL.getABITypeAlign(ValTy);
2307
2308 unsigned Size = DL.getTypeStoreSize(ValTy);
2309
2310 if (IsVolatile)
2312 if (IsNonTemporal)
2314 if (IsDereferenceable)
2316 if (IsInvariant)
2318
2319 return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2320 *Alignment, AAInfo, Ranges);
2321}
2322
2324 // If both operands are the same, then try to optimize or fold the cmp.
2325 CmpInst::Predicate Predicate = CI->getPredicate();
2326 if (CI->getOperand(0) != CI->getOperand(1))
2327 return Predicate;
2328
2329 switch (Predicate) {
2330 default: llvm_unreachable("Invalid predicate!");
2331 case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2332 case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2333 case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2334 case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2335 case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2336 case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2337 case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2338 case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2339 case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2340 case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2341 case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2342 case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2343 case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2344 case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2345 case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2346 case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2347
2348 case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2349 case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2350 case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2351 case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2352 case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2353 case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2354 case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2355 case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2356 case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2357 case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2358 }
2359
2360 return Predicate;
2361}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
uint64_t Size
static Register findLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:160
static bool isRegUsedByPhiNodes(Register DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:177
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:936
This file defines the FastISel class.
Hexagon Common GEP
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
This file contains the declarations for metadata subclasses.
#define P(N)
@ SI
@ VI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallPtrSet class.
This file defines the SmallString class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This file describes how to lower LLVM code to machine code.
@ Flags
Definition: TextStubV5.cpp:93
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
bool getValueAsBool() const
Return the attribute's value as a boolean.
Definition: Attributes.cpp:303
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:314
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:372
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:127
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1467
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1328
Value * getCalledOperand() const
Definition: InstrTypes.h:1401
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1353
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1334
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1266
unsigned arg_size() const
Definition: InstrTypes.h:1351
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:708
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:718
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:721
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:735
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:747
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:748
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:724
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:733
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:722
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:723
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:742
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:741
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:745
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:732
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:726
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:729
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:743
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:730
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:725
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:727
@ ICMP_EQ
equal
Definition: InstrTypes.h:739
@ ICMP_NE
not equal
Definition: InstrTypes.h:740
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:746
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:734
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:744
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:731
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:720
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:728
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:808
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:260
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:356
DWARF expression.
std::pair< DIExpression *, const ConstantInt * > constantFold(const ConstantInt *CI)
Try to shorten an expression with an initial constant operand.
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:698
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:861
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:848
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:500
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:669
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:468
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
This instruction extracts a struct member or array element value from an aggregate value.
ArrayRef< unsigned > getIndices() const
MachineRegisterInfo & MRI
Definition: FastISel.h:205
Register fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2109
const TargetLibraryInfo * LibInfo
Definition: FastISel.h:214
const DataLayout & DL
Definition: FastISel.h:210
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:531
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:237
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:644
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1642
DenseMap< const Value *, Register > LocalValueMap
Definition: FastISel.h:202
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor,...
Definition: FastISel.cpp:1561
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic.
Definition: FastISel.h:476
MachineInstr * EmitStartPt
The top most instruction in the current block that is allowed for emitting local variables.
Definition: FastISel.h:226
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:894
Register fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:1917
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering.
Definition: FastISel.cpp:1828
MachineInstr * getLastLocalValue()
Return the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:233
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1108
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1852
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:436
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:958
Register fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0)
Emit a MachineInstr with one register operand and a result register in the given register class.
Definition: FastISel.cpp:1926
bool selectFreeze(const User *I)
Definition: FastISel.cpp:1437
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1186
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1377
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2198
Register getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value.
Definition: FastISel.cpp:238
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:410
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:123
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2276
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:300
Register fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2092
MachineFrameInfo & MFI
Definition: FastISel.h:206
MachineFunction * MF
Definition: FastISel.h:204
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2259
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1832
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:69
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1477
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
Definition: FastISel.h:469
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1826
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:913
Register fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, unsigned Op1)
Emit a MachineInstr with two register operands and a result register in the given register class.
Definition: FastISel.cpp:1947
Register fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2021
Register createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:1896
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1847
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1824
bool selectFNeg(const User *I, const Value *In)
Emit an FNeg operation.
Definition: FastISel.cpp:1600
const TargetInstrInfo & TII
Definition: FastISel.h:211
bool selectCall(const User *I)
Definition: FastISel.cpp:1149
Register lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:351
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2323
void finishBasicBlock()
Flush the local value map.
Definition: FastISel.cpp:136
Register getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:383
MachineConstantPool & MCP
Definition: FastISel.h:207
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr),...
Definition: FastISel.cpp:1682
bool SkipTargetIndependentISel
Definition: FastISel.h:215
Register fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2046
Register constrainOperandRegClass(const MCInstrDesc &II, Register Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:1900
void updateValueMap(const Value *I, Register Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:362
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:444
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1811
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:756
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:401
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
const TargetLowering & TLI
Definition: FastISel.h:212
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1838
const TargetMachine & TM
Definition: FastISel.h:209
Register fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1861
Register fastEmitZExtFromI1(MVT VT, unsigned Op0)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero.
Definition: FastISel.cpp:2123
MIMetadata MIMD
Definition: FastISel.h:208
MachineInstr * LastLocalValue
The position of the last instruction for materializing constants for use in the current block.
Definition: FastISel.h:221
Register fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, unsigned Op1, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2066
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:138
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:430
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1581
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1408
Register fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:1998
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1834
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
Definition: FastISel.h:472
virtual ~FastISel()
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1843
const TargetRegisterInfo & TRI
Definition: FastISel.h:213
Register fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, unsigned Op1, unsigned Op2)
Emit a MachineInstr with three register operands and a result register in the given register class.
Definition: FastISel.cpp:1971
TargetLoweringBase::ArgListEntry ArgListEntry
Definition: FastISel.h:68
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Class to represent function types.
Definition: DerivedTypes.h:103
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:670
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:358
const BasicBlock * getParent() const
Definition: Instruction.h:90
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:87
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:275
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
Class to represent integer types.
Definition: DerivedTypes.h:40
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:331
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
An instruction for reading from memory.
Definition: Instructions.h:177
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:214
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:201
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:237
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:247
ArrayRef< MCPhysReg > implicit_defs() const
Return a list of registers that are potentially written by any instance of this machine instruction.
Definition: MCInstrDesc.h:577
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:943
Set of metadata that should be preserved when using BuildMI().
const DebugLoc & getDL() const
Machine Value Type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
MachineInstrBundleIterator< MachineInstr > iterator
void setHasStackMap(bool s=true)
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
Representation of each machine instruction.
Definition: MachineInstr.h:68
void setHeapAllocMarker(MachineFunction &MF, MDNode *MD)
Set a marker on instructions that denotes where we should create and emit heap alloc site labels.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void setPhysRegsDeadExcept(ArrayRef< Register > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
reg_iterator reg_begin(Register RegNo) const
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable's name.
Definition: Mangler.cpp:119
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
size_type size() const
Definition: SmallPtrSet.h:93
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:648
Class to represent struct types.
Definition: DerivedTypes.h:213
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise).
unsigned getCallFrameDestroyOpcode() const
virtual const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const
Returns a 0 terminated array of registers that can be safely used as scratch registers.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
const Triple & getTargetTriple() const
TargetOptions Options
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:356
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:638
bool isOSAIX() const
Tests whether the OS is AIX.
Definition: Triple.h:670
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ AnyReg
Used for dynamic register based calls (e.g.
Definition: CallingConv.h:60
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:786
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:923
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:832
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1145
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
Definition: ISDOpcodes.h:1339
Reg
All possible values of the reg field in the ModR/M byte.
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:226
reverse_iterator rend(StringRef path)
Get reverse end iterator over path.
Definition: Path.cpp:306
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void diagnoseDontCall(const CallInst &CI)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:748
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:297
gep_type_iterator gep_type_end(const User *GEP)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:388
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
@ Add
Sum of integers.
gep_type_iterator gep_type_begin(const User *GEP)
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:523
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:121
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:651
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:221
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:267
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:283
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:615
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:95
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:94
SmallVector< Register, 16 > OutRegs
Definition: FastISel.h:96
CallLoweringInfo & setTailCall(bool Value=true)
Definition: FastISel.h:177
SmallVector< Register, 4 > InRegs
Definition: FastISel.h:98
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:182
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, const CallBase &Call)
Definition: FastISel.h:104
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:97
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
This class contains a discriminated union of information about pointers in memory operands,...
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117