LLVM 19.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
21#include "llvm/Analysis/Loads.h"
52#include "llvm/IR/BasicBlock.h"
53#include "llvm/IR/CFG.h"
54#include "llvm/IR/Constant.h"
55#include "llvm/IR/Constants.h"
56#include "llvm/IR/DataLayout.h"
59#include "llvm/IR/Function.h"
61#include "llvm/IR/InlineAsm.h"
62#include "llvm/IR/InstrTypes.h"
65#include "llvm/IR/Intrinsics.h"
66#include "llvm/IR/IntrinsicsAMDGPU.h"
67#include "llvm/IR/LLVMContext.h"
68#include "llvm/IR/Metadata.h"
70#include "llvm/IR/Statepoint.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
75#include "llvm/MC/MCContext.h"
76#include "llvm/Pass.h"
79#include "llvm/Support/Debug.h"
87#include <algorithm>
88#include <cassert>
89#include <cstdint>
90#include <iterator>
91#include <optional>
92#include <string>
93#include <utility>
94#include <vector>
95
96#define DEBUG_TYPE "irtranslator"
97
98using namespace llvm;
99
100static cl::opt<bool>
101 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
102 cl::desc("Should enable CSE in irtranslator"),
103 cl::Optional, cl::init(false));
104char IRTranslator::ID = 0;
105
106INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
107 false, false)
115
120 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
121
122 // Print the function name explicitly if we don't have a debug location (which
123 // makes the diagnostic less useful) or if we're going to emit a raw error.
124 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
125 R << (" (in function: " + MF.getName() + ")").str();
126
127 if (TPC.isGlobalISelAbortEnabled())
128 report_fatal_error(Twine(R.getMsg()));
129 else
130 ORE.emit(R);
131}
132
134 : MachineFunctionPass(ID), OptLevel(optlevel) {}
135
136#ifndef NDEBUG
137namespace {
138/// Verify that every instruction created has the same DILocation as the
139/// instruction being translated.
140class DILocationVerifier : public GISelChangeObserver {
141 const Instruction *CurrInst = nullptr;
142
143public:
144 DILocationVerifier() = default;
145 ~DILocationVerifier() = default;
146
147 const Instruction *getCurrentInst() const { return CurrInst; }
148 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
149
150 void erasingInstr(MachineInstr &MI) override {}
151 void changingInstr(MachineInstr &MI) override {}
152 void changedInstr(MachineInstr &MI) override {}
153
154 void createdInstr(MachineInstr &MI) override {
155 assert(getCurrentInst() && "Inserted instruction without a current MI");
156
157 // Only print the check message if we're actually checking it.
158#ifndef NDEBUG
159 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
160 << " was copied to " << MI);
161#endif
162 // We allow insts in the entry block to have no debug loc because
163 // they could have originated from constants, and we don't want a jumpy
164 // debug experience.
165 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
166 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
167 (MI.isDebugInstr())) &&
168 "Line info was not transferred to all instructions");
169 }
170};
171} // namespace
172#endif // ifndef NDEBUG
173
174
180 if (OptLevel != CodeGenOptLevel::None) {
183 }
188}
189
191IRTranslator::allocateVRegs(const Value &Val) {
192 auto VRegsIt = VMap.findVRegs(Val);
193 if (VRegsIt != VMap.vregs_end())
194 return *VRegsIt->second;
195 auto *Regs = VMap.getVRegs(Val);
196 auto *Offsets = VMap.getOffsets(Val);
197 SmallVector<LLT, 4> SplitTys;
198 computeValueLLTs(*DL, *Val.getType(), SplitTys,
199 Offsets->empty() ? Offsets : nullptr);
200 for (unsigned i = 0; i < SplitTys.size(); ++i)
201 Regs->push_back(0);
202 return *Regs;
203}
204
205ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
206 auto VRegsIt = VMap.findVRegs(Val);
207 if (VRegsIt != VMap.vregs_end())
208 return *VRegsIt->second;
209
210 if (Val.getType()->isVoidTy())
211 return *VMap.getVRegs(Val);
212
213 // Create entry for this type.
214 auto *VRegs = VMap.getVRegs(Val);
215 auto *Offsets = VMap.getOffsets(Val);
216
217 assert(Val.getType()->isSized() &&
218 "Don't know how to create an empty vreg");
219
220 SmallVector<LLT, 4> SplitTys;
221 computeValueLLTs(*DL, *Val.getType(), SplitTys,
222 Offsets->empty() ? Offsets : nullptr);
223
224 if (!isa<Constant>(Val)) {
225 for (auto Ty : SplitTys)
226 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
227 return *VRegs;
228 }
229
230 if (Val.getType()->isAggregateType()) {
231 // UndefValue, ConstantAggregateZero
232 auto &C = cast<Constant>(Val);
233 unsigned Idx = 0;
234 while (auto Elt = C.getAggregateElement(Idx++)) {
235 auto EltRegs = getOrCreateVRegs(*Elt);
236 llvm::copy(EltRegs, std::back_inserter(*VRegs));
237 }
238 } else {
239 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
240 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
241 bool Success = translate(cast<Constant>(Val), VRegs->front());
242 if (!Success) {
243 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
245 &MF->getFunction().getEntryBlock());
246 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
247 reportTranslationError(*MF, *TPC, *ORE, R);
248 return *VRegs;
249 }
250 }
251
252 return *VRegs;
253}
254
255int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
256 auto MapEntry = FrameIndices.find(&AI);
257 if (MapEntry != FrameIndices.end())
258 return MapEntry->second;
259
260 uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
261 uint64_t Size =
262 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
263
264 // Always allocate at least one byte.
265 Size = std::max<uint64_t>(Size, 1u);
266
267 int &FI = FrameIndices[&AI];
268 FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
269 return FI;
270}
271
272Align IRTranslator::getMemOpAlign(const Instruction &I) {
273 if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
274 return SI->getAlign();
275 if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
276 return LI->getAlign();
277 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
278 return AI->getAlign();
279 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
280 return AI->getAlign();
281
282 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
283 R << "unable to translate memop: " << ore::NV("Opcode", &I);
284 reportTranslationError(*MF, *TPC, *ORE, R);
285 return Align(1);
286}
287
288MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
289 MachineBasicBlock *&MBB = BBToMBB[&BB];
290 assert(MBB && "BasicBlock was not encountered before");
291 return *MBB;
292}
293
294void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
295 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
296 MachinePreds[Edge].push_back(NewPred);
297}
298
299bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
300 MachineIRBuilder &MIRBuilder) {
301 // Get or create a virtual register for each value.
302 // Unless the value is a Constant => loadimm cst?
303 // or inline constant each time?
304 // Creation of a virtual register needs to have a size.
305 Register Op0 = getOrCreateVReg(*U.getOperand(0));
306 Register Op1 = getOrCreateVReg(*U.getOperand(1));
307 Register Res = getOrCreateVReg(U);
308 uint32_t Flags = 0;
309 if (isa<Instruction>(U)) {
310 const Instruction &I = cast<Instruction>(U);
312 }
313
314 MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
315 return true;
316}
317
318bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
319 MachineIRBuilder &MIRBuilder) {
320 Register Op0 = getOrCreateVReg(*U.getOperand(0));
321 Register Res = getOrCreateVReg(U);
322 uint32_t Flags = 0;
323 if (isa<Instruction>(U)) {
324 const Instruction &I = cast<Instruction>(U);
326 }
327 MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
328 return true;
329}
330
331bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
332 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
333}
334
335bool IRTranslator::translateCompare(const User &U,
336 MachineIRBuilder &MIRBuilder) {
337 auto *CI = dyn_cast<CmpInst>(&U);
338 Register Op0 = getOrCreateVReg(*U.getOperand(0));
339 Register Op1 = getOrCreateVReg(*U.getOperand(1));
340 Register Res = getOrCreateVReg(U);
341 CmpInst::Predicate Pred =
342 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
343 cast<ConstantExpr>(U).getPredicate());
344 if (CmpInst::isIntPredicate(Pred))
345 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
346 else if (Pred == CmpInst::FCMP_FALSE)
347 MIRBuilder.buildCopy(
348 Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
349 else if (Pred == CmpInst::FCMP_TRUE)
350 MIRBuilder.buildCopy(
351 Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
352 else {
353 uint32_t Flags = 0;
354 if (CI)
356 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
357 }
358
359 return true;
360}
361
362bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
363 const ReturnInst &RI = cast<ReturnInst>(U);
364 const Value *Ret = RI.getReturnValue();
365 if (Ret && DL->getTypeStoreSize(Ret->getType()).isZero())
366 Ret = nullptr;
367
368 ArrayRef<Register> VRegs;
369 if (Ret)
370 VRegs = getOrCreateVRegs(*Ret);
371
372 Register SwiftErrorVReg = 0;
373 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
374 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
375 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
376 }
377
378 // The target may mess up with the insertion point, but
379 // this is not important as a return is the last instruction
380 // of the block anyway.
381 return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
382}
383
384void IRTranslator::emitBranchForMergedCondition(
386 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
387 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
388 // If the leaf of the tree is a comparison, merge the condition into
389 // the caseblock.
390 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
391 CmpInst::Predicate Condition;
392 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
393 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
394 } else {
395 const FCmpInst *FC = cast<FCmpInst>(Cond);
396 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
397 }
398
399 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
400 BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
401 CurBuilder->getDebugLoc(), TProb, FProb);
402 SL->SwitchCases.push_back(CB);
403 return;
404 }
405
406 // Create a CaseBlock record representing this branch.
409 Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
410 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
411 SL->SwitchCases.push_back(CB);
412}
413
414static bool isValInBlock(const Value *V, const BasicBlock *BB) {
415 if (const Instruction *I = dyn_cast<Instruction>(V))
416 return I->getParent() == BB;
417 return true;
418}
419
420void IRTranslator::findMergedConditions(
422 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
424 BranchProbability FProb, bool InvertCond) {
425 using namespace PatternMatch;
426 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
427 "Expected Opc to be AND/OR");
428 // Skip over not part of the tree and remember to invert op and operands at
429 // next level.
430 Value *NotCond;
431 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
432 isValInBlock(NotCond, CurBB->getBasicBlock())) {
433 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
434 !InvertCond);
435 return;
436 }
437
438 const Instruction *BOp = dyn_cast<Instruction>(Cond);
439 const Value *BOpOp0, *BOpOp1;
440 // Compute the effective opcode for Cond, taking into account whether it needs
441 // to be inverted, e.g.
442 // and (not (or A, B)), C
443 // gets lowered as
444 // and (and (not A, not B), C)
446 if (BOp) {
447 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
448 ? Instruction::And
449 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
450 ? Instruction::Or
452 if (InvertCond) {
453 if (BOpc == Instruction::And)
454 BOpc = Instruction::Or;
455 else if (BOpc == Instruction::Or)
456 BOpc = Instruction::And;
457 }
458 }
459
460 // If this node is not part of the or/and tree, emit it as a branch.
461 // Note that all nodes in the tree should have same opcode.
462 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
463 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
464 !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
465 !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
466 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
467 InvertCond);
468 return;
469 }
470
471 // Create TmpBB after CurBB.
472 MachineFunction::iterator BBI(CurBB);
473 MachineBasicBlock *TmpBB =
475 CurBB->getParent()->insert(++BBI, TmpBB);
476
477 if (Opc == Instruction::Or) {
478 // Codegen X | Y as:
479 // BB1:
480 // jmp_if_X TBB
481 // jmp TmpBB
482 // TmpBB:
483 // jmp_if_Y TBB
484 // jmp FBB
485 //
486
487 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
488 // The requirement is that
489 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
490 // = TrueProb for original BB.
491 // Assuming the original probabilities are A and B, one choice is to set
492 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
493 // A/(1+B) and 2B/(1+B). This choice assumes that
494 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
495 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
496 // TmpBB, but the math is more complicated.
497
498 auto NewTrueProb = TProb / 2;
499 auto NewFalseProb = TProb / 2 + FProb;
500 // Emit the LHS condition.
501 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
502 NewFalseProb, InvertCond);
503
504 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
505 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
506 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
507 // Emit the RHS condition into TmpBB.
508 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
509 Probs[1], InvertCond);
510 } else {
511 assert(Opc == Instruction::And && "Unknown merge op!");
512 // Codegen X & Y as:
513 // BB1:
514 // jmp_if_X TmpBB
515 // jmp FBB
516 // TmpBB:
517 // jmp_if_Y TBB
518 // jmp FBB
519 //
520 // This requires creation of TmpBB after CurBB.
521
522 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
523 // The requirement is that
524 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
525 // = FalseProb for original BB.
526 // Assuming the original probabilities are A and B, one choice is to set
527 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
528 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
529 // TrueProb for BB1 * FalseProb for TmpBB.
530
531 auto NewTrueProb = TProb + FProb / 2;
532 auto NewFalseProb = FProb / 2;
533 // Emit the LHS condition.
534 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
535 NewFalseProb, InvertCond);
536
537 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
538 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
539 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
540 // Emit the RHS condition into TmpBB.
541 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
542 Probs[1], InvertCond);
543 }
544}
545
546bool IRTranslator::shouldEmitAsBranches(
547 const std::vector<SwitchCG::CaseBlock> &Cases) {
548 // For multiple cases, it's better to emit as branches.
549 if (Cases.size() != 2)
550 return true;
551
552 // If this is two comparisons of the same values or'd or and'd together, they
553 // will get folded into a single comparison, so don't emit two blocks.
554 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
555 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
556 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
557 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
558 return false;
559 }
560
561 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
562 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
563 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
564 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
565 isa<Constant>(Cases[0].CmpRHS) &&
566 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
567 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
568 Cases[0].TrueBB == Cases[1].ThisBB)
569 return false;
570 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
571 Cases[0].FalseBB == Cases[1].ThisBB)
572 return false;
573 }
574
575 return true;
576}
577
578bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
579 const BranchInst &BrInst = cast<BranchInst>(U);
580 auto &CurMBB = MIRBuilder.getMBB();
581 auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
582
583 if (BrInst.isUnconditional()) {
584 // If the unconditional target is the layout successor, fallthrough.
585 if (OptLevel == CodeGenOptLevel::None ||
586 !CurMBB.isLayoutSuccessor(Succ0MBB))
587 MIRBuilder.buildBr(*Succ0MBB);
588
589 // Link successors.
590 for (const BasicBlock *Succ : successors(&BrInst))
591 CurMBB.addSuccessor(&getMBB(*Succ));
592 return true;
593 }
594
595 // If this condition is one of the special cases we handle, do special stuff
596 // now.
597 const Value *CondVal = BrInst.getCondition();
598 MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
599
600 // If this is a series of conditions that are or'd or and'd together, emit
601 // this as a sequence of branches instead of setcc's with and/or operations.
602 // As long as jumps are not expensive (exceptions for multi-use logic ops,
603 // unpredictable branches, and vector extracts because those jumps are likely
604 // expensive for any target), this should improve performance.
605 // For example, instead of something like:
606 // cmp A, B
607 // C = seteq
608 // cmp D, E
609 // F = setle
610 // or C, F
611 // jnz foo
612 // Emit:
613 // cmp A, B
614 // je foo
615 // cmp D, E
616 // jle foo
617 using namespace PatternMatch;
618 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
619 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
620 !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
622 Value *Vec;
623 const Value *BOp0, *BOp1;
624 if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
625 Opcode = Instruction::And;
626 else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
627 Opcode = Instruction::Or;
628
629 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
630 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
631 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
632 getEdgeProbability(&CurMBB, Succ0MBB),
633 getEdgeProbability(&CurMBB, Succ1MBB),
634 /*InvertCond=*/false);
635 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
636
637 // Allow some cases to be rejected.
638 if (shouldEmitAsBranches(SL->SwitchCases)) {
639 // Emit the branch for this block.
640 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
641 SL->SwitchCases.erase(SL->SwitchCases.begin());
642 return true;
643 }
644
645 // Okay, we decided not to do this, remove any inserted MBB's and clear
646 // SwitchCases.
647 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
648 MF->erase(SL->SwitchCases[I].ThisBB);
649
650 SL->SwitchCases.clear();
651 }
652 }
653
654 // Create a CaseBlock record representing this branch.
655 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
657 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
658 CurBuilder->getDebugLoc());
659
660 // Use emitSwitchCase to actually insert the fast branch sequence for this
661 // cond branch.
662 emitSwitchCase(CB, &CurMBB, *CurBuilder);
663 return true;
664}
665
666void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
668 BranchProbability Prob) {
669 if (!FuncInfo.BPI) {
670 Src->addSuccessorWithoutProb(Dst);
671 return;
672 }
673 if (Prob.isUnknown())
674 Prob = getEdgeProbability(Src, Dst);
675 Src->addSuccessor(Dst, Prob);
676}
677
679IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
680 const MachineBasicBlock *Dst) const {
681 const BasicBlock *SrcBB = Src->getBasicBlock();
682 const BasicBlock *DstBB = Dst->getBasicBlock();
683 if (!FuncInfo.BPI) {
684 // If BPI is not available, set the default probability as 1 / N, where N is
685 // the number of successors.
686 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
687 return BranchProbability(1, SuccSize);
688 }
689 return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
690}
691
692bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
693 using namespace SwitchCG;
694 // Extract cases from the switch.
695 const SwitchInst &SI = cast<SwitchInst>(U);
696 BranchProbabilityInfo *BPI = FuncInfo.BPI;
697 CaseClusterVector Clusters;
698 Clusters.reserve(SI.getNumCases());
699 for (const auto &I : SI.cases()) {
700 MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
701 assert(Succ && "Could not find successor mbb in mapping");
702 const ConstantInt *CaseVal = I.getCaseValue();
703 BranchProbability Prob =
704 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
705 : BranchProbability(1, SI.getNumCases() + 1);
706 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
707 }
708
709 MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
710
711 // Cluster adjacent cases with the same destination. We do this at all
712 // optimization levels because it's cheap to do and will make codegen faster
713 // if there are many clusters.
714 sortAndRangeify(Clusters);
715
716 MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
717
718 // If there is only the default destination, jump there directly.
719 if (Clusters.empty()) {
720 SwitchMBB->addSuccessor(DefaultMBB);
721 if (DefaultMBB != SwitchMBB->getNextNode())
722 MIB.buildBr(*DefaultMBB);
723 return true;
724 }
725
726 SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB, nullptr, nullptr);
727 SL->findBitTestClusters(Clusters, &SI);
728
729 LLVM_DEBUG({
730 dbgs() << "Case clusters: ";
731 for (const CaseCluster &C : Clusters) {
732 if (C.Kind == CC_JumpTable)
733 dbgs() << "JT:";
734 if (C.Kind == CC_BitTests)
735 dbgs() << "BT:";
736
737 C.Low->getValue().print(dbgs(), true);
738 if (C.Low != C.High) {
739 dbgs() << '-';
740 C.High->getValue().print(dbgs(), true);
741 }
742 dbgs() << ' ';
743 }
744 dbgs() << '\n';
745 });
746
747 assert(!Clusters.empty());
748 SwitchWorkList WorkList;
749 CaseClusterIt First = Clusters.begin();
750 CaseClusterIt Last = Clusters.end() - 1;
751 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
752 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
753
754 while (!WorkList.empty()) {
755 SwitchWorkListItem W = WorkList.pop_back_val();
756
757 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
758 // For optimized builds, lower large range as a balanced binary tree.
759 if (NumClusters > 3 &&
761 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
762 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB, MIB);
763 continue;
764 }
765
766 if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
767 return false;
768 }
769 return true;
770}
771
772void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
774 Value *Cond, MachineBasicBlock *SwitchMBB,
775 MachineIRBuilder &MIB) {
776 using namespace SwitchCG;
777 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
778 "Clusters not sorted?");
779 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
780
781 auto [LastLeft, FirstRight, LeftProb, RightProb] =
782 SL->computeSplitWorkItemInfo(W);
783
784 // Use the first element on the right as pivot since we will make less-than
785 // comparisons against it.
786 CaseClusterIt PivotCluster = FirstRight;
787 assert(PivotCluster > W.FirstCluster);
788 assert(PivotCluster <= W.LastCluster);
789
790 CaseClusterIt FirstLeft = W.FirstCluster;
791 CaseClusterIt LastRight = W.LastCluster;
792
793 const ConstantInt *Pivot = PivotCluster->Low;
794
795 // New blocks will be inserted immediately after the current one.
797 ++BBI;
798
799 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
800 // we can branch to its destination directly if it's squeezed exactly in
801 // between the known lower bound and Pivot - 1.
802 MachineBasicBlock *LeftMBB;
803 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
804 FirstLeft->Low == W.GE &&
805 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
806 LeftMBB = FirstLeft->MBB;
807 } else {
808 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
809 FuncInfo.MF->insert(BBI, LeftMBB);
810 WorkList.push_back(
811 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
812 }
813
814 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
815 // single cluster, RHS.Low == Pivot, and we can branch to its destination
816 // directly if RHS.High equals the current upper bound.
817 MachineBasicBlock *RightMBB;
818 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
819 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
820 RightMBB = FirstRight->MBB;
821 } else {
822 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
823 FuncInfo.MF->insert(BBI, RightMBB);
824 WorkList.push_back(
825 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
826 }
827
828 // Create the CaseBlock record that will be used to lower the branch.
829 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
830 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
831 RightProb);
832
833 if (W.MBB == SwitchMBB)
834 emitSwitchCase(CB, SwitchMBB, MIB);
835 else
836 SL->SwitchCases.push_back(CB);
837}
838
839void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
841 // Emit the code for the jump table
842 assert(JT.Reg != -1U && "Should lower JT Header first!");
844 MIB.setMBB(*MBB);
845 MIB.setDebugLoc(CurBuilder->getDebugLoc());
846
848 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
849
850 auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
851 MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
852}
853
854bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
856 MachineBasicBlock *HeaderBB) {
857 MachineIRBuilder MIB(*HeaderBB->getParent());
858 MIB.setMBB(*HeaderBB);
859 MIB.setDebugLoc(CurBuilder->getDebugLoc());
860
861 const Value &SValue = *JTH.SValue;
862 // Subtract the lowest switch case value from the value being switched on.
863 const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
864 Register SwitchOpReg = getOrCreateVReg(SValue);
865 auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
866 auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
867
868 // This value may be smaller or larger than the target's pointer type, and
869 // therefore require extension or truncating.
870 auto *PtrIRTy = PointerType::getUnqual(SValue.getContext());
871 const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
872 Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
873
874 JT.Reg = Sub.getReg(0);
875
876 if (JTH.FallthroughUnreachable) {
877 if (JT.MBB != HeaderBB->getNextNode())
878 MIB.buildBr(*JT.MBB);
879 return true;
880 }
881
882 // Emit the range check for the jump table, and branch to the default block
883 // for the switch statement if the value being switched on exceeds the
884 // largest case in the switch.
885 auto Cst = getOrCreateVReg(
886 *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
887 Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
888 auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
889
890 auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
891
892 // Avoid emitting unnecessary branches to the next block.
893 if (JT.MBB != HeaderBB->getNextNode())
894 BrCond = MIB.buildBr(*JT.MBB);
895 return true;
896}
897
898void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
899 MachineBasicBlock *SwitchBB,
900 MachineIRBuilder &MIB) {
901 Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
903 DebugLoc OldDbgLoc = MIB.getDebugLoc();
904 MIB.setDebugLoc(CB.DbgLoc);
905 MIB.setMBB(*CB.ThisBB);
906
907 if (CB.PredInfo.NoCmp) {
908 // Branch or fall through to TrueBB.
909 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
910 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
911 CB.ThisBB);
913 if (CB.TrueBB != CB.ThisBB->getNextNode())
914 MIB.buildBr(*CB.TrueBB);
915 MIB.setDebugLoc(OldDbgLoc);
916 return;
917 }
918
919 const LLT i1Ty = LLT::scalar(1);
920 // Build the compare.
921 if (!CB.CmpMHS) {
922 const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
923 // For conditional branch lowering, we might try to do something silly like
924 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
925 // just re-use the existing condition vreg.
926 if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
928 Cond = CondLHS;
929 } else {
930 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
932 Cond =
933 MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
934 else
935 Cond =
936 MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
937 }
938 } else {
940 "Can only handle SLE ranges");
941
942 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
943 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
944
945 Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
946 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
947 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
948 Cond =
949 MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
950 } else {
951 const LLT CmpTy = MRI->getType(CmpOpReg);
952 auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
953 auto Diff = MIB.buildConstant(CmpTy, High - Low);
954 Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
955 }
956 }
957
958 // Update successor info
959 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
960
961 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
962 CB.ThisBB);
963
964 // TrueBB and FalseBB are always different unless the incoming IR is
965 // degenerate. This only happens when running llc on weird IR.
966 if (CB.TrueBB != CB.FalseBB)
967 addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
969
970 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
971 CB.ThisBB);
972
973 MIB.buildBrCond(Cond, *CB.TrueBB);
974 MIB.buildBr(*CB.FalseBB);
975 MIB.setDebugLoc(OldDbgLoc);
976}
977
978bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
979 MachineBasicBlock *SwitchMBB,
980 MachineBasicBlock *CurMBB,
981 MachineBasicBlock *DefaultMBB,
982 MachineIRBuilder &MIB,
984 BranchProbability UnhandledProbs,
986 MachineBasicBlock *Fallthrough,
987 bool FallthroughUnreachable) {
988 using namespace SwitchCG;
989 MachineFunction *CurMF = SwitchMBB->getParent();
990 // FIXME: Optimize away range check based on pivot comparisons.
991 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
992 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
993 BranchProbability DefaultProb = W.DefaultProb;
994
995 // The jump block hasn't been inserted yet; insert it here.
996 MachineBasicBlock *JumpMBB = JT->MBB;
997 CurMF->insert(BBI, JumpMBB);
998
999 // Since the jump table block is separate from the switch block, we need
1000 // to keep track of it as a machine predecessor to the default block,
1001 // otherwise we lose the phi edges.
1002 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1003 CurMBB);
1004 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1005 JumpMBB);
1006
1007 auto JumpProb = I->Prob;
1008 auto FallthroughProb = UnhandledProbs;
1009
1010 // If the default statement is a target of the jump table, we evenly
1011 // distribute the default probability to successors of CurMBB. Also
1012 // update the probability on the edge from JumpMBB to Fallthrough.
1013 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1014 SE = JumpMBB->succ_end();
1015 SI != SE; ++SI) {
1016 if (*SI == DefaultMBB) {
1017 JumpProb += DefaultProb / 2;
1018 FallthroughProb -= DefaultProb / 2;
1019 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
1020 JumpMBB->normalizeSuccProbs();
1021 } else {
1022 // Also record edges from the jump table block to it's successors.
1023 addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1024 JumpMBB);
1025 }
1026 }
1027
1028 if (FallthroughUnreachable)
1029 JTH->FallthroughUnreachable = true;
1030
1031 if (!JTH->FallthroughUnreachable)
1032 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
1033 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
1034 CurMBB->normalizeSuccProbs();
1035
1036 // The jump table header will be inserted in our current block, do the
1037 // range check, and fall through to our fallthrough block.
1038 JTH->HeaderBB = CurMBB;
1039 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1040
1041 // If we're in the right place, emit the jump table header right now.
1042 if (CurMBB == SwitchMBB) {
1043 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
1044 return false;
1045 JTH->Emitted = true;
1046 }
1047 return true;
1048}
1049bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1050 Value *Cond,
1051 MachineBasicBlock *Fallthrough,
1052 bool FallthroughUnreachable,
1053 BranchProbability UnhandledProbs,
1054 MachineBasicBlock *CurMBB,
1055 MachineIRBuilder &MIB,
1056 MachineBasicBlock *SwitchMBB) {
1057 using namespace SwitchCG;
1058 const Value *RHS, *LHS, *MHS;
1059 CmpInst::Predicate Pred;
1060 if (I->Low == I->High) {
1061 // Check Cond == I->Low.
1062 Pred = CmpInst::ICMP_EQ;
1063 LHS = Cond;
1064 RHS = I->Low;
1065 MHS = nullptr;
1066 } else {
1067 // Check I->Low <= Cond <= I->High.
1068 Pred = CmpInst::ICMP_SLE;
1069 LHS = I->Low;
1070 MHS = Cond;
1071 RHS = I->High;
1072 }
1073
1074 // If Fallthrough is unreachable, fold away the comparison.
1075 // The false probability is the sum of all unhandled cases.
1076 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1077 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1078
1079 emitSwitchCase(CB, SwitchMBB, MIB);
1080 return true;
1081}
1082
1083void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1084 MachineBasicBlock *SwitchBB) {
1085 MachineIRBuilder &MIB = *CurBuilder;
1086 MIB.setMBB(*SwitchBB);
1087
1088 // Subtract the minimum value.
1089 Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1090
1091 LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1092 Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1093 auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1094
1096 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1097
1098 LLT MaskTy = SwitchOpTy;
1099 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1100 !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
1101 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1102 else {
1103 // Ensure that the type will fit the mask value.
1104 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1105 if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1106 // Switch table case range are encoded into series of masks.
1107 // Just use pointer type, it's guaranteed to fit.
1108 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1109 break;
1110 }
1111 }
1112 }
1113 Register SubReg = RangeSub.getReg(0);
1114 if (SwitchOpTy != MaskTy)
1115 SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1116
1117 B.RegVT = getMVTForLLT(MaskTy);
1118 B.Reg = SubReg;
1119
1120 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1121
1122 if (!B.FallthroughUnreachable)
1123 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1124 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1125
1126 SwitchBB->normalizeSuccProbs();
1127
1128 if (!B.FallthroughUnreachable) {
1129 // Conditional branch to the default block.
1130 auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1131 auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1132 RangeSub, RangeCst);
1133 MIB.buildBrCond(RangeCmp, *B.Default);
1134 }
1135
1136 // Avoid emitting unnecessary branches to the next block.
1137 if (MBB != SwitchBB->getNextNode())
1138 MIB.buildBr(*MBB);
1139}
1140
1141void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1142 MachineBasicBlock *NextMBB,
1143 BranchProbability BranchProbToNext,
1145 MachineBasicBlock *SwitchBB) {
1146 MachineIRBuilder &MIB = *CurBuilder;
1147 MIB.setMBB(*SwitchBB);
1148
1149 LLT SwitchTy = getLLTForMVT(BB.RegVT);
1150 Register Cmp;
1151 unsigned PopCount = llvm::popcount(B.Mask);
1152 if (PopCount == 1) {
1153 // Testing for a single bit; just compare the shift count with what it
1154 // would need to be to shift a 1 bit in that position.
1155 auto MaskTrailingZeros =
1156 MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
1157 Cmp =
1158 MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1159 .getReg(0);
1160 } else if (PopCount == BB.Range) {
1161 // There is only one zero bit in the range, test for it directly.
1162 auto MaskTrailingOnes =
1163 MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
1164 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1165 .getReg(0);
1166 } else {
1167 // Make desired shift.
1168 auto CstOne = MIB.buildConstant(SwitchTy, 1);
1169 auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1170
1171 // Emit bit tests and jumps.
1172 auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1173 auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1174 auto CstZero = MIB.buildConstant(SwitchTy, 0);
1175 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1176 .getReg(0);
1177 }
1178
1179 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1180 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1181 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1182 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1183 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1184 // one as they are relative probabilities (and thus work more like weights),
1185 // and hence we need to normalize them to let the sum of them become one.
1186 SwitchBB->normalizeSuccProbs();
1187
1188 // Record the fact that the IR edge from the header to the bit test target
1189 // will go through our new block. Neeeded for PHIs to have nodes added.
1190 addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1191 SwitchBB);
1192
1193 MIB.buildBrCond(Cmp, *B.TargetBB);
1194
1195 // Avoid emitting unnecessary branches to the next block.
1196 if (NextMBB != SwitchBB->getNextNode())
1197 MIB.buildBr(*NextMBB);
1198}
1199
1200bool IRTranslator::lowerBitTestWorkItem(
1202 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1204 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1206 bool FallthroughUnreachable) {
1207 using namespace SwitchCG;
1208 MachineFunction *CurMF = SwitchMBB->getParent();
1209 // FIXME: Optimize away range check based on pivot comparisons.
1210 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1211 // The bit test blocks haven't been inserted yet; insert them here.
1212 for (BitTestCase &BTC : BTB->Cases)
1213 CurMF->insert(BBI, BTC.ThisBB);
1214
1215 // Fill in fields of the BitTestBlock.
1216 BTB->Parent = CurMBB;
1217 BTB->Default = Fallthrough;
1218
1219 BTB->DefaultProb = UnhandledProbs;
1220 // If the cases in bit test don't form a contiguous range, we evenly
1221 // distribute the probability on the edge to Fallthrough to two
1222 // successors of CurMBB.
1223 if (!BTB->ContiguousRange) {
1224 BTB->Prob += DefaultProb / 2;
1225 BTB->DefaultProb -= DefaultProb / 2;
1226 }
1227
1228 if (FallthroughUnreachable)
1229 BTB->FallthroughUnreachable = true;
1230
1231 // If we're in the right place, emit the bit test header right now.
1232 if (CurMBB == SwitchMBB) {
1233 emitBitTestHeader(*BTB, SwitchMBB);
1234 BTB->Emitted = true;
1235 }
1236 return true;
1237}
1238
1239bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1240 Value *Cond,
1241 MachineBasicBlock *SwitchMBB,
1242 MachineBasicBlock *DefaultMBB,
1243 MachineIRBuilder &MIB) {
1244 using namespace SwitchCG;
1245 MachineFunction *CurMF = FuncInfo.MF;
1246 MachineBasicBlock *NextMBB = nullptr;
1248 if (++BBI != FuncInfo.MF->end())
1249 NextMBB = &*BBI;
1250
1251 if (EnableOpts) {
1252 // Here, we order cases by probability so the most likely case will be
1253 // checked first. However, two clusters can have the same probability in
1254 // which case their relative ordering is non-deterministic. So we use Low
1255 // as a tie-breaker as clusters are guaranteed to never overlap.
1256 llvm::sort(W.FirstCluster, W.LastCluster + 1,
1257 [](const CaseCluster &a, const CaseCluster &b) {
1258 return a.Prob != b.Prob
1259 ? a.Prob > b.Prob
1260 : a.Low->getValue().slt(b.Low->getValue());
1261 });
1262
1263 // Rearrange the case blocks so that the last one falls through if possible
1264 // without changing the order of probabilities.
1265 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1266 --I;
1267 if (I->Prob > W.LastCluster->Prob)
1268 break;
1269 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1270 std::swap(*I, *W.LastCluster);
1271 break;
1272 }
1273 }
1274 }
1275
1276 // Compute total probability.
1277 BranchProbability DefaultProb = W.DefaultProb;
1278 BranchProbability UnhandledProbs = DefaultProb;
1279 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1280 UnhandledProbs += I->Prob;
1281
1282 MachineBasicBlock *CurMBB = W.MBB;
1283 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1284 bool FallthroughUnreachable = false;
1285 MachineBasicBlock *Fallthrough;
1286 if (I == W.LastCluster) {
1287 // For the last cluster, fall through to the default destination.
1288 Fallthrough = DefaultMBB;
1289 FallthroughUnreachable = isa<UnreachableInst>(
1290 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1291 } else {
1292 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1293 CurMF->insert(BBI, Fallthrough);
1294 }
1295 UnhandledProbs -= I->Prob;
1296
1297 switch (I->Kind) {
1298 case CC_BitTests: {
1299 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1300 DefaultProb, UnhandledProbs, I, Fallthrough,
1301 FallthroughUnreachable)) {
1302 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1303 return false;
1304 }
1305 break;
1306 }
1307
1308 case CC_JumpTable: {
1309 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1310 UnhandledProbs, I, Fallthrough,
1311 FallthroughUnreachable)) {
1312 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1313 return false;
1314 }
1315 break;
1316 }
1317 case CC_Range: {
1318 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1319 FallthroughUnreachable, UnhandledProbs,
1320 CurMBB, MIB, SwitchMBB)) {
1321 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1322 return false;
1323 }
1324 break;
1325 }
1326 }
1327 CurMBB = Fallthrough;
1328 }
1329
1330 return true;
1331}
1332
1333bool IRTranslator::translateIndirectBr(const User &U,
1334 MachineIRBuilder &MIRBuilder) {
1335 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1336
1337 const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1338 MIRBuilder.buildBrIndirect(Tgt);
1339
1340 // Link successors.
1342 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1343 for (const BasicBlock *Succ : successors(&BrInst)) {
1344 // It's legal for indirectbr instructions to have duplicate blocks in the
1345 // destination list. We don't allow this in MIR. Skip anything that's
1346 // already a successor.
1347 if (!AddedSuccessors.insert(Succ).second)
1348 continue;
1349 CurBB.addSuccessor(&getMBB(*Succ));
1350 }
1351
1352 return true;
1353}
1354
1355static bool isSwiftError(const Value *V) {
1356 if (auto Arg = dyn_cast<Argument>(V))
1357 return Arg->hasSwiftErrorAttr();
1358 if (auto AI = dyn_cast<AllocaInst>(V))
1359 return AI->isSwiftError();
1360 return false;
1361}
1362
1363bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1364 const LoadInst &LI = cast<LoadInst>(U);
1365
1366 unsigned StoreSize = DL->getTypeStoreSize(LI.getType());
1367 if (StoreSize == 0)
1368 return true;
1369
1370 ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1371 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1372 Register Base = getOrCreateVReg(*LI.getPointerOperand());
1373 AAMDNodes AAInfo = LI.getAAMetadata();
1374
1375 const Value *Ptr = LI.getPointerOperand();
1376 Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
1377 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1378
1379 if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
1380 assert(Regs.size() == 1 && "swifterror should be single pointer");
1381 Register VReg =
1382 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1383 MIRBuilder.buildCopy(Regs[0], VReg);
1384 return true;
1385 }
1386
1388 TLI->getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
1389 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1390 if (AA->pointsToConstantMemory(
1391 MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
1393 }
1394 }
1395
1396 const MDNode *Ranges =
1397 Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1398 for (unsigned i = 0; i < Regs.size(); ++i) {
1399 Register Addr;
1400 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1401
1402 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1403 Align BaseAlign = getMemOpAlign(LI);
1404 auto MMO = MF->getMachineMemOperand(
1405 Ptr, Flags, MRI->getType(Regs[i]),
1406 commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
1407 LI.getSyncScopeID(), LI.getOrdering());
1408 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1409 }
1410
1411 return true;
1412}
1413
1414bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1415 const StoreInst &SI = cast<StoreInst>(U);
1416 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1417 return true;
1418
1419 ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1420 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1421 Register Base = getOrCreateVReg(*SI.getPointerOperand());
1422
1423 Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
1424 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1425
1426 if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1427 assert(Vals.size() == 1 && "swifterror should be single pointer");
1428
1429 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1430 SI.getPointerOperand());
1431 MIRBuilder.buildCopy(VReg, Vals[0]);
1432 return true;
1433 }
1434
1436
1437 for (unsigned i = 0; i < Vals.size(); ++i) {
1438 Register Addr;
1439 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1440
1441 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1442 Align BaseAlign = getMemOpAlign(SI);
1443 auto MMO = MF->getMachineMemOperand(
1444 Ptr, Flags, MRI->getType(Vals[i]),
1445 commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1446 SI.getSyncScopeID(), SI.getOrdering());
1447 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1448 }
1449 return true;
1450}
1451
1453 const Value *Src = U.getOperand(0);
1454 Type *Int32Ty = Type::getInt32Ty(U.getContext());
1455
1456 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1457 // usual array element rather than looking into the actual aggregate.
1459 Indices.push_back(ConstantInt::get(Int32Ty, 0));
1460
1461 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1462 for (auto Idx : EVI->indices())
1463 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1464 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1465 for (auto Idx : IVI->indices())
1466 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1467 } else {
1468 for (unsigned i = 1; i < U.getNumOperands(); ++i)
1469 Indices.push_back(U.getOperand(i));
1470 }
1471
1472 return 8 * static_cast<uint64_t>(
1473 DL.getIndexedOffsetInType(Src->getType(), Indices));
1474}
1475
1476bool IRTranslator::translateExtractValue(const User &U,
1477 MachineIRBuilder &MIRBuilder) {
1478 const Value *Src = U.getOperand(0);
1480 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1481 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1482 unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1483 auto &DstRegs = allocateVRegs(U);
1484
1485 for (unsigned i = 0; i < DstRegs.size(); ++i)
1486 DstRegs[i] = SrcRegs[Idx++];
1487
1488 return true;
1489}
1490
1491bool IRTranslator::translateInsertValue(const User &U,
1492 MachineIRBuilder &MIRBuilder) {
1493 const Value *Src = U.getOperand(0);
1495 auto &DstRegs = allocateVRegs(U);
1496 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1497 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1498 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1499 auto *InsertedIt = InsertedRegs.begin();
1500
1501 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1502 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1503 DstRegs[i] = *InsertedIt++;
1504 else
1505 DstRegs[i] = SrcRegs[i];
1506 }
1507
1508 return true;
1509}
1510
1511bool IRTranslator::translateSelect(const User &U,
1512 MachineIRBuilder &MIRBuilder) {
1513 Register Tst = getOrCreateVReg(*U.getOperand(0));
1514 ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1515 ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1516 ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1517
1518 uint32_t Flags = 0;
1519 if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1521
1522 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1523 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1524 }
1525
1526 return true;
1527}
1528
1529bool IRTranslator::translateCopy(const User &U, const Value &V,
1530 MachineIRBuilder &MIRBuilder) {
1531 Register Src = getOrCreateVReg(V);
1532 auto &Regs = *VMap.getVRegs(U);
1533 if (Regs.empty()) {
1534 Regs.push_back(Src);
1535 VMap.getOffsets(U)->push_back(0);
1536 } else {
1537 // If we already assigned a vreg for this instruction, we can't change that.
1538 // Emit a copy to satisfy the users we already emitted.
1539 MIRBuilder.buildCopy(Regs[0], Src);
1540 }
1541 return true;
1542}
1543
1544bool IRTranslator::translateBitCast(const User &U,
1545 MachineIRBuilder &MIRBuilder) {
1546 // If we're bitcasting to the source type, we can reuse the source vreg.
1547 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1548 getLLTForType(*U.getType(), *DL)) {
1549 // If the source is a ConstantInt then it was probably created by
1550 // ConstantHoisting and we should leave it alone.
1551 if (isa<ConstantInt>(U.getOperand(0)))
1552 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1553 MIRBuilder);
1554 return translateCopy(U, *U.getOperand(0), MIRBuilder);
1555 }
1556
1557 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1558}
1559
1560bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1561 MachineIRBuilder &MIRBuilder) {
1562 if (U.getType()->getScalarType()->isBFloatTy() ||
1563 U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
1564 return false;
1565 Register Op = getOrCreateVReg(*U.getOperand(0));
1566 Register Res = getOrCreateVReg(U);
1567 MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1568 return true;
1569}
1570
1571bool IRTranslator::translateGetElementPtr(const User &U,
1572 MachineIRBuilder &MIRBuilder) {
1573 Value &Op0 = *U.getOperand(0);
1574 Register BaseReg = getOrCreateVReg(Op0);
1575 Type *PtrIRTy = Op0.getType();
1576 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1577 Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
1578 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1579
1580 uint32_t Flags = 0;
1581 if (isa<Instruction>(U)) {
1582 const Instruction &I = cast<Instruction>(U);
1584 }
1585
1586 // Normalize Vector GEP - all scalar operands should be converted to the
1587 // splat vector.
1588 unsigned VectorWidth = 0;
1589
1590 // True if we should use a splat vector; using VectorWidth alone is not
1591 // sufficient.
1592 bool WantSplatVector = false;
1593 if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1594 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1595 // We don't produce 1 x N vectors; those are treated as scalars.
1596 WantSplatVector = VectorWidth > 1;
1597 }
1598
1599 // We might need to splat the base pointer into a vector if the offsets
1600 // are vectors.
1601 if (WantSplatVector && !PtrTy.isVector()) {
1602 BaseReg = MIRBuilder
1603 .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy),
1604 BaseReg)
1605 .getReg(0);
1606 PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1607 PtrTy = getLLTForType(*PtrIRTy, *DL);
1608 OffsetIRTy = DL->getIndexType(PtrIRTy);
1609 OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1610 }
1611
1612 int64_t Offset = 0;
1613 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1614 GTI != E; ++GTI) {
1615 const Value *Idx = GTI.getOperand();
1616 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1617 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1619 continue;
1620 } else {
1621 uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
1622
1623 // If this is a scalar constant or a splat vector of constants,
1624 // handle it quickly.
1625 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1626 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1627 Offset += ElementSize * *Val;
1628 continue;
1629 }
1630 }
1631
1632 if (Offset != 0) {
1633 auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1634 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1635 .getReg(0);
1636 Offset = 0;
1637 }
1638
1639 Register IdxReg = getOrCreateVReg(*Idx);
1640 LLT IdxTy = MRI->getType(IdxReg);
1641 if (IdxTy != OffsetTy) {
1642 if (!IdxTy.isVector() && WantSplatVector) {
1643 IdxReg = MIRBuilder
1645 IdxReg)
1646 .getReg(0);
1647 }
1648
1649 IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1650 }
1651
1652 // N = N + Idx * ElementSize;
1653 // Avoid doing it for ElementSize of 1.
1654 Register GepOffsetReg;
1655 if (ElementSize != 1) {
1656 auto ElementSizeMIB = MIRBuilder.buildConstant(
1657 getLLTForType(*OffsetIRTy, *DL), ElementSize);
1658 GepOffsetReg =
1659 MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1660 } else
1661 GepOffsetReg = IdxReg;
1662
1663 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1664 }
1665 }
1666
1667 if (Offset != 0) {
1668 auto OffsetMIB =
1669 MIRBuilder.buildConstant(OffsetTy, Offset);
1670
1671 if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
1673
1674 MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
1675 Flags);
1676 return true;
1677 }
1678
1679 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1680 return true;
1681}
1682
1683bool IRTranslator::translateMemFunc(const CallInst &CI,
1684 MachineIRBuilder &MIRBuilder,
1685 unsigned Opcode) {
1686 const Value *SrcPtr = CI.getArgOperand(1);
1687 // If the source is undef, then just emit a nop.
1688 if (isa<UndefValue>(SrcPtr))
1689 return true;
1690
1692
1693 unsigned MinPtrSize = UINT_MAX;
1694 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1695 Register SrcReg = getOrCreateVReg(**AI);
1696 LLT SrcTy = MRI->getType(SrcReg);
1697 if (SrcTy.isPointer())
1698 MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1699 SrcRegs.push_back(SrcReg);
1700 }
1701
1702 LLT SizeTy = LLT::scalar(MinPtrSize);
1703
1704 // The size operand should be the minimum of the pointer sizes.
1705 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1706 if (MRI->getType(SizeOpReg) != SizeTy)
1707 SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1708
1709 auto ICall = MIRBuilder.buildInstr(Opcode);
1710 for (Register SrcReg : SrcRegs)
1711 ICall.addUse(SrcReg);
1712
1713 Align DstAlign;
1714 Align SrcAlign;
1715 unsigned IsVol =
1716 cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1717
1718 ConstantInt *CopySize = nullptr;
1719
1720 if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1721 DstAlign = MCI->getDestAlign().valueOrOne();
1722 SrcAlign = MCI->getSourceAlign().valueOrOne();
1723 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1724 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1725 DstAlign = MCI->getDestAlign().valueOrOne();
1726 SrcAlign = MCI->getSourceAlign().valueOrOne();
1727 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1728 } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1729 DstAlign = MMI->getDestAlign().valueOrOne();
1730 SrcAlign = MMI->getSourceAlign().valueOrOne();
1731 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1732 } else {
1733 auto *MSI = cast<MemSetInst>(&CI);
1734 DstAlign = MSI->getDestAlign().valueOrOne();
1735 }
1736
1737 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1738 // We need to propagate the tail call flag from the IR inst as an argument.
1739 // Otherwise, we have to pessimize and assume later that we cannot tail call
1740 // any memory intrinsics.
1741 ICall.addImm(CI.isTailCall() ? 1 : 0);
1742 }
1743
1744 // Create mem operands to store the alignment and volatile info.
1747 if (IsVol) {
1748 LoadFlags |= MachineMemOperand::MOVolatile;
1749 StoreFlags |= MachineMemOperand::MOVolatile;
1750 }
1751
1752 AAMDNodes AAInfo = CI.getAAMetadata();
1753 if (AA && CopySize &&
1755 SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
1756 LoadFlags |= MachineMemOperand::MOInvariant;
1757
1758 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1759 // but the previous usage implied it did. Probably should check
1760 // isDereferenceableAndAlignedPointer.
1762 }
1763
1764 ICall.addMemOperand(
1766 StoreFlags, 1, DstAlign, AAInfo));
1767 if (Opcode != TargetOpcode::G_MEMSET)
1768 ICall.addMemOperand(MF->getMachineMemOperand(
1769 MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
1770
1771 return true;
1772}
1773
1774bool IRTranslator::translateVectorInterleave2Intrinsic(
1775 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1776 assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 &&
1777 "This function can only be called on the interleave2 intrinsic!");
1778 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1779 Register Op0 = getOrCreateVReg(*CI.getOperand(0));
1780 Register Op1 = getOrCreateVReg(*CI.getOperand(1));
1781 Register Res = getOrCreateVReg(CI);
1782
1783 LLT OpTy = MRI->getType(Op0);
1784 MIRBuilder.buildShuffleVector(Res, Op0, Op1,
1786
1787 return true;
1788}
1789
1790bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1791 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1792 assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_deinterleave2 &&
1793 "This function can only be called on the deinterleave2 intrinsic!");
1794 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1795 // SelectionDAG).
1796 Register Op = getOrCreateVReg(*CI.getOperand(0));
1797 auto Undef = MIRBuilder.buildUndef(MRI->getType(Op));
1798 ArrayRef<Register> Res = getOrCreateVRegs(CI);
1799
1800 LLT ResTy = MRI->getType(Res[0]);
1801 MIRBuilder.buildShuffleVector(Res[0], Op, Undef,
1802 createStrideMask(0, 2, ResTy.getNumElements()));
1803 MIRBuilder.buildShuffleVector(Res[1], Op, Undef,
1804 createStrideMask(1, 2, ResTy.getNumElements()));
1805
1806 return true;
1807}
1808
1809void IRTranslator::getStackGuard(Register DstReg,
1810 MachineIRBuilder &MIRBuilder) {
1812 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1813 auto MIB =
1814 MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1815
1817 if (!Global)
1818 return;
1819
1820 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1821 LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1822
1823 MachinePointerInfo MPInfo(Global);
1827 MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1828 MIB.setMemRefs({MemRef});
1829}
1830
1831bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1832 MachineIRBuilder &MIRBuilder) {
1833 ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1834 MIRBuilder.buildInstr(
1835 Op, {ResRegs[0], ResRegs[1]},
1836 {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1837
1838 return true;
1839}
1840
1841bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1842 MachineIRBuilder &MIRBuilder) {
1843 Register Dst = getOrCreateVReg(CI);
1844 Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1845 Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1846 uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1847 MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1848 return true;
1849}
1850
1851unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1852 switch (ID) {
1853 default:
1854 break;
1855 case Intrinsic::bswap:
1856 return TargetOpcode::G_BSWAP;
1857 case Intrinsic::bitreverse:
1858 return TargetOpcode::G_BITREVERSE;
1859 case Intrinsic::fshl:
1860 return TargetOpcode::G_FSHL;
1861 case Intrinsic::fshr:
1862 return TargetOpcode::G_FSHR;
1863 case Intrinsic::ceil:
1864 return TargetOpcode::G_FCEIL;
1865 case Intrinsic::cos:
1866 return TargetOpcode::G_FCOS;
1867 case Intrinsic::ctpop:
1868 return TargetOpcode::G_CTPOP;
1869 case Intrinsic::exp:
1870 return TargetOpcode::G_FEXP;
1871 case Intrinsic::exp2:
1872 return TargetOpcode::G_FEXP2;
1873 case Intrinsic::exp10:
1874 return TargetOpcode::G_FEXP10;
1875 case Intrinsic::fabs:
1876 return TargetOpcode::G_FABS;
1877 case Intrinsic::copysign:
1878 return TargetOpcode::G_FCOPYSIGN;
1879 case Intrinsic::minnum:
1880 return TargetOpcode::G_FMINNUM;
1881 case Intrinsic::maxnum:
1882 return TargetOpcode::G_FMAXNUM;
1883 case Intrinsic::minimum:
1884 return TargetOpcode::G_FMINIMUM;
1885 case Intrinsic::maximum:
1886 return TargetOpcode::G_FMAXIMUM;
1887 case Intrinsic::canonicalize:
1888 return TargetOpcode::G_FCANONICALIZE;
1889 case Intrinsic::floor:
1890 return TargetOpcode::G_FFLOOR;
1891 case Intrinsic::fma:
1892 return TargetOpcode::G_FMA;
1893 case Intrinsic::log:
1894 return TargetOpcode::G_FLOG;
1895 case Intrinsic::log2:
1896 return TargetOpcode::G_FLOG2;
1897 case Intrinsic::log10:
1898 return TargetOpcode::G_FLOG10;
1899 case Intrinsic::ldexp:
1900 return TargetOpcode::G_FLDEXP;
1901 case Intrinsic::nearbyint:
1902 return TargetOpcode::G_FNEARBYINT;
1903 case Intrinsic::pow:
1904 return TargetOpcode::G_FPOW;
1905 case Intrinsic::powi:
1906 return TargetOpcode::G_FPOWI;
1907 case Intrinsic::rint:
1908 return TargetOpcode::G_FRINT;
1909 case Intrinsic::round:
1910 return TargetOpcode::G_INTRINSIC_ROUND;
1911 case Intrinsic::roundeven:
1912 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1913 case Intrinsic::sin:
1914 return TargetOpcode::G_FSIN;
1915 case Intrinsic::sqrt:
1916 return TargetOpcode::G_FSQRT;
1917 case Intrinsic::trunc:
1918 return TargetOpcode::G_INTRINSIC_TRUNC;
1919 case Intrinsic::readcyclecounter:
1920 return TargetOpcode::G_READCYCLECOUNTER;
1921 case Intrinsic::readsteadycounter:
1922 return TargetOpcode::G_READSTEADYCOUNTER;
1923 case Intrinsic::ptrmask:
1924 return TargetOpcode::G_PTRMASK;
1925 case Intrinsic::lrint:
1926 return TargetOpcode::G_INTRINSIC_LRINT;
1927 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1928 case Intrinsic::vector_reduce_fmin:
1929 return TargetOpcode::G_VECREDUCE_FMIN;
1930 case Intrinsic::vector_reduce_fmax:
1931 return TargetOpcode::G_VECREDUCE_FMAX;
1932 case Intrinsic::vector_reduce_fminimum:
1933 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1934 case Intrinsic::vector_reduce_fmaximum:
1935 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1936 case Intrinsic::vector_reduce_add:
1937 return TargetOpcode::G_VECREDUCE_ADD;
1938 case Intrinsic::vector_reduce_mul:
1939 return TargetOpcode::G_VECREDUCE_MUL;
1940 case Intrinsic::vector_reduce_and:
1941 return TargetOpcode::G_VECREDUCE_AND;
1942 case Intrinsic::vector_reduce_or:
1943 return TargetOpcode::G_VECREDUCE_OR;
1944 case Intrinsic::vector_reduce_xor:
1945 return TargetOpcode::G_VECREDUCE_XOR;
1946 case Intrinsic::vector_reduce_smax:
1947 return TargetOpcode::G_VECREDUCE_SMAX;
1948 case Intrinsic::vector_reduce_smin:
1949 return TargetOpcode::G_VECREDUCE_SMIN;
1950 case Intrinsic::vector_reduce_umax:
1951 return TargetOpcode::G_VECREDUCE_UMAX;
1952 case Intrinsic::vector_reduce_umin:
1953 return TargetOpcode::G_VECREDUCE_UMIN;
1954 case Intrinsic::lround:
1955 return TargetOpcode::G_LROUND;
1956 case Intrinsic::llround:
1957 return TargetOpcode::G_LLROUND;
1958 case Intrinsic::get_fpenv:
1959 return TargetOpcode::G_GET_FPENV;
1960 case Intrinsic::get_fpmode:
1961 return TargetOpcode::G_GET_FPMODE;
1962 }
1964}
1965
1966bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1968 MachineIRBuilder &MIRBuilder) {
1969
1970 unsigned Op = getSimpleIntrinsicOpcode(ID);
1971
1972 // Is this a simple intrinsic?
1974 return false;
1975
1976 // Yes. Let's translate it.
1978 for (const auto &Arg : CI.args())
1979 VRegs.push_back(getOrCreateVReg(*Arg));
1980
1981 MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1983 return true;
1984}
1985
1986// TODO: Include ConstainedOps.def when all strict instructions are defined.
1988 switch (ID) {
1989 case Intrinsic::experimental_constrained_fadd:
1990 return TargetOpcode::G_STRICT_FADD;
1991 case Intrinsic::experimental_constrained_fsub:
1992 return TargetOpcode::G_STRICT_FSUB;
1993 case Intrinsic::experimental_constrained_fmul:
1994 return TargetOpcode::G_STRICT_FMUL;
1995 case Intrinsic::experimental_constrained_fdiv:
1996 return TargetOpcode::G_STRICT_FDIV;
1997 case Intrinsic::experimental_constrained_frem:
1998 return TargetOpcode::G_STRICT_FREM;
1999 case Intrinsic::experimental_constrained_fma:
2000 return TargetOpcode::G_STRICT_FMA;
2001 case Intrinsic::experimental_constrained_sqrt:
2002 return TargetOpcode::G_STRICT_FSQRT;
2003 case Intrinsic::experimental_constrained_ldexp:
2004 return TargetOpcode::G_STRICT_FLDEXP;
2005 default:
2006 return 0;
2007 }
2008}
2009
2010bool IRTranslator::translateConstrainedFPIntrinsic(
2011 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2013
2014 unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
2015 if (!Opcode)
2016 return false;
2017
2021
2023 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
2024 if (!FPI.isUnaryOp())
2025 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
2026 if (FPI.isTernaryOp())
2027 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
2028
2029 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
2030 return true;
2031}
2032
2033std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2034 auto VRegs = getOrCreateVRegs(Arg);
2035 if (VRegs.size() != 1)
2036 return std::nullopt;
2037
2038 // Arguments are lowered as a copy of a livein physical register.
2039 auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
2040 if (!VRegDef || !VRegDef->isCopy())
2041 return std::nullopt;
2042 return VRegDef->getOperand(1).getReg().asMCReg();
2043}
2044
2045bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2046 const DILocalVariable *Var,
2047 const DIExpression *Expr,
2048 const DebugLoc &DL,
2049 MachineIRBuilder &MIRBuilder) {
2050 auto *Arg = dyn_cast<Argument>(Val);
2051 if (!Arg)
2052 return false;
2053
2054 if (!Expr->isEntryValue())
2055 return false;
2056
2057 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
2058 if (!PhysReg) {
2059 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2060 << ": expression is entry_value but "
2061 << "couldn't find a physical register\n");
2062 LLVM_DEBUG(dbgs() << *Var << "\n");
2063 return true;
2064 }
2065
2066 if (isDeclare) {
2067 // Append an op deref to account for the fact that this is a dbg_declare.
2068 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
2069 MF->setVariableDbgInfo(Var, Expr, *PhysReg, DL);
2070 } else {
2071 MIRBuilder.buildDirectDbgValue(*PhysReg, Var, Expr);
2072 }
2073
2074 return true;
2075}
2076
2077bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2078 MachineIRBuilder &MIRBuilder) {
2079 if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
2080 if (ORE->enabled()) {
2081 if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
2082 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2083 R.visit(MI);
2084 }
2085 }
2086 }
2087
2088 // If this is a simple intrinsic (that is, we just need to add a def of
2089 // a vreg, and uses for each arg operand, then translate it.
2090 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2091 return true;
2092
2093 switch (ID) {
2094 default:
2095 break;
2096 case Intrinsic::lifetime_start:
2097 case Intrinsic::lifetime_end: {
2098 // No stack colouring in O0, discard region information.
2100 return true;
2101
2102 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2103 : TargetOpcode::LIFETIME_END;
2104
2105 // Get the underlying objects for the location passed on the lifetime
2106 // marker.
2108 getUnderlyingObjects(CI.getArgOperand(1), Allocas);
2109
2110 // Iterate over each underlying object, creating lifetime markers for each
2111 // static alloca. Quit if we find a non-static alloca.
2112 for (const Value *V : Allocas) {
2113 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
2114 if (!AI)
2115 continue;
2116
2117 if (!AI->isStaticAlloca())
2118 return true;
2119
2120 MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
2121 }
2122 return true;
2123 }
2124 case Intrinsic::dbg_declare: {
2125 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
2126 assert(DI.getVariable() && "Missing variable");
2127 translateDbgDeclareRecord(DI.getAddress(), DI.hasArgList(), DI.getVariable(),
2128 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2129 return true;
2130 }
2131 case Intrinsic::dbg_label: {
2132 const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
2133 assert(DI.getLabel() && "Missing label");
2134
2136 MIRBuilder.getDebugLoc()) &&
2137 "Expected inlined-at fields to agree");
2138
2139 MIRBuilder.buildDbgLabel(DI.getLabel());
2140 return true;
2141 }
2142 case Intrinsic::vaend:
2143 // No target I know of cares about va_end. Certainly no in-tree target
2144 // does. Simplest intrinsic ever!
2145 return true;
2146 case Intrinsic::vastart: {
2147 Value *Ptr = CI.getArgOperand(0);
2148 unsigned ListSize = TLI->getVaListSizeInBits(*DL) / 8;
2149 Align Alignment = getKnownAlignment(Ptr, *DL);
2150
2151 MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
2152 .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
2154 ListSize, Alignment));
2155 return true;
2156 }
2157 case Intrinsic::dbg_assign:
2158 // A dbg.assign is a dbg.value with more information about stack locations,
2159 // typically produced during optimisation of variables with leaked
2160 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2161 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2162 // need to register for and use the AssignmentTrackingAnalysis pass.
2164 case Intrinsic::dbg_value: {
2165 // This form of DBG_VALUE is target-independent.
2166 const DbgValueInst &DI = cast<DbgValueInst>(CI);
2167 translateDbgValueRecord(DI.getValue(), DI.hasArgList(), DI.getVariable(),
2168 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2169 return true;
2170 }
2171 case Intrinsic::uadd_with_overflow:
2172 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2173 case Intrinsic::sadd_with_overflow:
2174 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2175 case Intrinsic::usub_with_overflow:
2176 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2177 case Intrinsic::ssub_with_overflow:
2178 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2179 case Intrinsic::umul_with_overflow:
2180 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2181 case Intrinsic::smul_with_overflow:
2182 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2183 case Intrinsic::uadd_sat:
2184 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2185 case Intrinsic::sadd_sat:
2186 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2187 case Intrinsic::usub_sat:
2188 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2189 case Intrinsic::ssub_sat:
2190 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2191 case Intrinsic::ushl_sat:
2192 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2193 case Intrinsic::sshl_sat:
2194 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2195 case Intrinsic::umin:
2196 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2197 case Intrinsic::umax:
2198 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2199 case Intrinsic::smin:
2200 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2201 case Intrinsic::smax:
2202 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2203 case Intrinsic::abs:
2204 // TODO: Preserve "int min is poison" arg in GMIR?
2205 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2206 case Intrinsic::smul_fix:
2207 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2208 case Intrinsic::umul_fix:
2209 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2210 case Intrinsic::smul_fix_sat:
2211 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2212 case Intrinsic::umul_fix_sat:
2213 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2214 case Intrinsic::sdiv_fix:
2215 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2216 case Intrinsic::udiv_fix:
2217 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2218 case Intrinsic::sdiv_fix_sat:
2219 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2220 case Intrinsic::udiv_fix_sat:
2221 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2222 case Intrinsic::fmuladd: {
2223 const TargetMachine &TM = MF->getTarget();
2224 Register Dst = getOrCreateVReg(CI);
2225 Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2226 Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2227 Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2228 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2230 TLI->getValueType(*DL, CI.getType()))) {
2231 // TODO: Revisit this to see if we should move this part of the
2232 // lowering to the combiner.
2233 MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2235 } else {
2236 LLT Ty = getLLTForType(*CI.getType(), *DL);
2237 auto FMul = MIRBuilder.buildFMul(
2238 Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2239 MIRBuilder.buildFAdd(Dst, FMul, Op2,
2241 }
2242 return true;
2243 }
2244 case Intrinsic::convert_from_fp16:
2245 // FIXME: This intrinsic should probably be removed from the IR.
2246 MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2247 getOrCreateVReg(*CI.getArgOperand(0)),
2249 return true;
2250 case Intrinsic::convert_to_fp16:
2251 // FIXME: This intrinsic should probably be removed from the IR.
2252 MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2253 getOrCreateVReg(*CI.getArgOperand(0)),
2255 return true;
2256 case Intrinsic::frexp: {
2257 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2258 MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
2259 getOrCreateVReg(*CI.getArgOperand(0)),
2261 return true;
2262 }
2263 case Intrinsic::memcpy_inline:
2264 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2265 case Intrinsic::memcpy:
2266 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2267 case Intrinsic::memmove:
2268 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2269 case Intrinsic::memset:
2270 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2271 case Intrinsic::eh_typeid_for: {
2273 Register Reg = getOrCreateVReg(CI);
2274 unsigned TypeID = MF->getTypeIDFor(GV);
2275 MIRBuilder.buildConstant(Reg, TypeID);
2276 return true;
2277 }
2278 case Intrinsic::objectsize:
2279 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2280
2281 case Intrinsic::is_constant:
2282 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2283
2284 case Intrinsic::stackguard:
2285 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2286 return true;
2287 case Intrinsic::stackprotector: {
2288 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2289 Register GuardVal;
2290 if (TLI->useLoadStackGuardNode()) {
2291 GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2292 getStackGuard(GuardVal, MIRBuilder);
2293 } else
2294 GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
2295
2296 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2297 int FI = getOrCreateFrameIndex(*Slot);
2299
2300 MIRBuilder.buildStore(
2301 GuardVal, getOrCreateVReg(*Slot),
2305 PtrTy, Align(8)));
2306 return true;
2307 }
2308 case Intrinsic::stacksave: {
2309 MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2310 return true;
2311 }
2312 case Intrinsic::stackrestore: {
2313 MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},
2314 {getOrCreateVReg(*CI.getArgOperand(0))});
2315 return true;
2316 }
2317 case Intrinsic::cttz:
2318 case Intrinsic::ctlz: {
2319 ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2320 bool isTrailing = ID == Intrinsic::cttz;
2321 unsigned Opcode = isTrailing
2322 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2323 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2324 : Cst->isZero() ? TargetOpcode::G_CTLZ
2325 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2326 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2327 {getOrCreateVReg(*CI.getArgOperand(0))});
2328 return true;
2329 }
2330 case Intrinsic::invariant_start: {
2331 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2333 MIRBuilder.buildUndef(Undef);
2334 return true;
2335 }
2336 case Intrinsic::invariant_end:
2337 return true;
2338 case Intrinsic::expect:
2339 case Intrinsic::annotation:
2340 case Intrinsic::ptr_annotation:
2341 case Intrinsic::launder_invariant_group:
2342 case Intrinsic::strip_invariant_group: {
2343 // Drop the intrinsic, but forward the value.
2344 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2345 getOrCreateVReg(*CI.getArgOperand(0)));
2346 return true;
2347 }
2348 case Intrinsic::assume:
2349 case Intrinsic::experimental_noalias_scope_decl:
2350 case Intrinsic::var_annotation:
2351 case Intrinsic::sideeffect:
2352 // Discard annotate attributes, assumptions, and artificial side-effects.
2353 return true;
2354 case Intrinsic::read_volatile_register:
2355 case Intrinsic::read_register: {
2356 Value *Arg = CI.getArgOperand(0);
2357 MIRBuilder
2358 .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2359 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2360 return true;
2361 }
2362 case Intrinsic::write_register: {
2363 Value *Arg = CI.getArgOperand(0);
2364 MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2365 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2366 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2367 return true;
2368 }
2369 case Intrinsic::localescape: {
2370 MachineBasicBlock &EntryMBB = MF->front();
2372
2373 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2374 // is the same on all targets.
2375 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2377 if (isa<ConstantPointerNull>(Arg))
2378 continue; // Skip null pointers. They represent a hole in index space.
2379
2380 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2381 MCSymbol *FrameAllocSym =
2382 MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2383 Idx);
2384
2385 // This should be inserted at the start of the entry block.
2386 auto LocalEscape =
2387 MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2388 .addSym(FrameAllocSym)
2389 .addFrameIndex(FI);
2390
2391 EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2392 }
2393
2394 return true;
2395 }
2396 case Intrinsic::vector_reduce_fadd:
2397 case Intrinsic::vector_reduce_fmul: {
2398 // Need to check for the reassoc flag to decide whether we want a
2399 // sequential reduction opcode or not.
2400 Register Dst = getOrCreateVReg(CI);
2401 Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2402 Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2403 unsigned Opc = 0;
2404 if (!CI.hasAllowReassoc()) {
2405 // The sequential ordering case.
2406 Opc = ID == Intrinsic::vector_reduce_fadd
2407 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2408 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2409 MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2411 return true;
2412 }
2413 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2414 // since the associativity doesn't matter.
2415 unsigned ScalarOpc;
2416 if (ID == Intrinsic::vector_reduce_fadd) {
2417 Opc = TargetOpcode::G_VECREDUCE_FADD;
2418 ScalarOpc = TargetOpcode::G_FADD;
2419 } else {
2420 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2421 ScalarOpc = TargetOpcode::G_FMUL;
2422 }
2423 LLT DstTy = MRI->getType(Dst);
2424 auto Rdx = MIRBuilder.buildInstr(
2425 Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2426 MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2428
2429 return true;
2430 }
2431 case Intrinsic::trap:
2432 case Intrinsic::debugtrap:
2433 case Intrinsic::ubsantrap: {
2434 StringRef TrapFuncName =
2435 CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
2436 if (TrapFuncName.empty())
2437 break; // Use the default handling.
2439 if (ID == Intrinsic::ubsantrap) {
2440 Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
2441 CI.getArgOperand(0)->getType(), 0});
2442 }
2443 Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
2444 Info.CB = &CI;
2445 Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
2446 return CLI->lowerCall(MIRBuilder, Info);
2447 }
2448 case Intrinsic::amdgcn_cs_chain:
2449 return translateCallBase(CI, MIRBuilder);
2450 case Intrinsic::fptrunc_round: {
2452
2453 // Convert the metadata argument to a constant integer
2454 Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
2455 std::optional<RoundingMode> RoundMode =
2456 convertStrToRoundingMode(cast<MDString>(MD)->getString());
2457
2458 // Add the Rounding mode as an integer
2459 MIRBuilder
2460 .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2461 {getOrCreateVReg(CI)},
2462 {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
2463 .addImm((int)*RoundMode);
2464
2465 return true;
2466 }
2467 case Intrinsic::is_fpclass: {
2468 Value *FpValue = CI.getOperand(0);
2469 ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
2470
2471 MIRBuilder
2472 .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2473 {getOrCreateVReg(*FpValue)})
2474 .addImm(TestMaskValue->getZExtValue());
2475
2476 return true;
2477 }
2478 case Intrinsic::set_fpenv: {
2479 Value *FPEnv = CI.getOperand(0);
2480 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPENV, {},
2481 {getOrCreateVReg(*FPEnv)});
2482 return true;
2483 }
2484 case Intrinsic::reset_fpenv: {
2485 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPENV, {}, {});
2486 return true;
2487 }
2488 case Intrinsic::set_fpmode: {
2489 Value *FPState = CI.getOperand(0);
2490 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPMODE, {},
2491 { getOrCreateVReg(*FPState) });
2492 return true;
2493 }
2494 case Intrinsic::reset_fpmode: {
2495 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
2496 return true;
2497 }
2498 case Intrinsic::prefetch: {
2499 Value *Addr = CI.getOperand(0);
2500 unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
2501 unsigned Locality = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
2502 unsigned CacheType = cast<ConstantInt>(CI.getOperand(3))->getZExtValue();
2503
2505 auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags,
2506 LLT(), Align());
2507
2508 MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType,
2509 MMO);
2510
2511 return true;
2512 }
2513
2514 case Intrinsic::experimental_vector_interleave2:
2515 case Intrinsic::experimental_vector_deinterleave2: {
2516 // Both intrinsics have at least one operand.
2517 Value *Op0 = CI.getOperand(0);
2518 LLT ResTy = getLLTForType(*Op0->getType(), MIRBuilder.getDataLayout());
2519 if (!ResTy.isFixedVector())
2520 return false;
2521
2522 if (CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2)
2523 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2524
2525 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2526 }
2527
2528#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2529 case Intrinsic::INTRINSIC:
2530#include "llvm/IR/ConstrainedOps.def"
2531 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2532 MIRBuilder);
2533
2534 }
2535 return false;
2536}
2537
2538bool IRTranslator::translateInlineAsm(const CallBase &CB,
2539 MachineIRBuilder &MIRBuilder) {
2540
2542
2543 if (!ALI) {
2544 LLVM_DEBUG(
2545 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2546 return false;
2547 }
2548
2549 return ALI->lowerInlineAsm(
2550 MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2551}
2552
2553bool IRTranslator::translateCallBase(const CallBase &CB,
2554 MachineIRBuilder &MIRBuilder) {
2555 ArrayRef<Register> Res = getOrCreateVRegs(CB);
2556
2558 Register SwiftInVReg = 0;
2559 Register SwiftErrorVReg = 0;
2560 for (const auto &Arg : CB.args()) {
2561 if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2562 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2563 LLT Ty = getLLTForType(*Arg->getType(), *DL);
2564 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2565 MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2566 &CB, &MIRBuilder.getMBB(), Arg));
2567 Args.emplace_back(ArrayRef(SwiftInVReg));
2568 SwiftErrorVReg =
2569 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2570 continue;
2571 }
2572 Args.push_back(getOrCreateVRegs(*Arg));
2573 }
2574
2575 if (auto *CI = dyn_cast<CallInst>(&CB)) {
2576 if (ORE->enabled()) {
2577 if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
2578 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2579 R.visit(CI);
2580 }
2581 }
2582 }
2583
2584 // We don't set HasCalls on MFI here yet because call lowering may decide to
2585 // optimize into tail calls. Instead, we defer that to selection where a final
2586 // scan is done to check if any instructions are calls.
2587 bool Success =
2588 CLI->lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
2589 [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2590
2591 // Check if we just inserted a tail call.
2592 if (Success) {
2593 assert(!HasTailCall && "Can't tail call return twice from block?");
2595 HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2596 }
2597
2598 return Success;
2599}
2600
2601bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2602 const CallInst &CI = cast<CallInst>(U);
2603 auto TII = MF->getTarget().getIntrinsicInfo();
2604 const Function *F = CI.getCalledFunction();
2605
2606 // FIXME: support Windows dllimport function calls and calls through
2607 // weak symbols.
2608 if (F && (F->hasDLLImportStorageClass() ||
2610 F->hasExternalWeakLinkage())))
2611 return false;
2612
2613 // FIXME: support control flow guard targets.
2615 return false;
2616
2617 // FIXME: support statepoints and related.
2618 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2619 return false;
2620
2621 if (CI.isInlineAsm())
2622 return translateInlineAsm(CI, MIRBuilder);
2623
2624 diagnoseDontCall(CI);
2625
2627 if (F && F->isIntrinsic()) {
2628 ID = F->getIntrinsicID();
2630 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2631 }
2632
2633 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2634 return translateCallBase(CI, MIRBuilder);
2635
2636 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2637
2638 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2639 return true;
2640
2641 ArrayRef<Register> ResultRegs;
2642 if (!CI.getType()->isVoidTy())
2643 ResultRegs = getOrCreateVRegs(CI);
2644
2645 // Ignore the callsite attributes. Backend code is most likely not expecting
2646 // an intrinsic to sometimes have side effects and sometimes not.
2647 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
2648 if (isa<FPMathOperator>(CI))
2649 MIB->copyIRFlags(CI);
2650
2651 for (const auto &Arg : enumerate(CI.args())) {
2652 // If this is required to be an immediate, don't materialize it in a
2653 // register.
2654 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2655 if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2656 // imm arguments are more convenient than cimm (and realistically
2657 // probably sufficient), so use them.
2658 assert(CI->getBitWidth() <= 64 &&
2659 "large intrinsic immediates not handled");
2660 MIB.addImm(CI->getSExtValue());
2661 } else {
2662 MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2663 }
2664 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2665 auto *MD = MDVal->getMetadata();
2666 auto *MDN = dyn_cast<MDNode>(MD);
2667 if (!MDN) {
2668 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2669 MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2670 else // This was probably an MDString.
2671 return false;
2672 }
2673 MIB.addMetadata(MDN);
2674 } else {
2675 ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2676 if (VRegs.size() > 1)
2677 return false;
2678 MIB.addUse(VRegs[0]);
2679 }
2680 }
2681
2682 // Add a MachineMemOperand if it is a target mem intrinsic.
2684 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2685 if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2686 Align Alignment = Info.align.value_or(
2687 DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2688 LLT MemTy = Info.memVT.isSimple()
2689 ? getLLTForMVT(Info.memVT.getSimpleVT())
2690 : LLT::scalar(Info.memVT.getStoreSizeInBits());
2691
2692 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2693 // didn't yield anything useful.
2695 if (Info.ptrVal)
2696 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2697 else if (Info.fallbackAddressSpace)
2698 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2699 MIB.addMemOperand(
2700 MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
2701 }
2702
2703 return true;
2704}
2705
2706bool IRTranslator::findUnwindDestinations(
2707 const BasicBlock *EHPadBB,
2708 BranchProbability Prob,
2709 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2710 &UnwindDests) {
2712 EHPadBB->getParent()->getFunction().getPersonalityFn());
2713 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2714 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2715 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2716 bool IsSEH = isAsynchronousEHPersonality(Personality);
2717
2718 if (IsWasmCXX) {
2719 // Ignore this for now.
2720 return false;
2721 }
2722
2723 while (EHPadBB) {
2724 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2725 BasicBlock *NewEHPadBB = nullptr;
2726 if (isa<LandingPadInst>(Pad)) {
2727 // Stop on landingpads. They are not funclets.
2728 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2729 break;
2730 }
2731 if (isa<CleanupPadInst>(Pad)) {
2732 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2733 // personalities.
2734 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2735 UnwindDests.back().first->setIsEHScopeEntry();
2736 UnwindDests.back().first->setIsEHFuncletEntry();
2737 break;
2738 }
2739 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2740 // Add the catchpad handlers to the possible destinations.
2741 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2742 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2743 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2744 if (IsMSVCCXX || IsCoreCLR)
2745 UnwindDests.back().first->setIsEHFuncletEntry();
2746 if (!IsSEH)
2747 UnwindDests.back().first->setIsEHScopeEntry();
2748 }
2749 NewEHPadBB = CatchSwitch->getUnwindDest();
2750 } else {
2751 continue;
2752 }
2753
2754 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2755 if (BPI && NewEHPadBB)
2756 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2757 EHPadBB = NewEHPadBB;
2758 }
2759 return true;
2760}
2761
2762bool IRTranslator::translateInvoke(const User &U,
2763 MachineIRBuilder &MIRBuilder) {
2764 const InvokeInst &I = cast<InvokeInst>(U);
2765 MCContext &Context = MF->getContext();
2766
2767 const BasicBlock *ReturnBB = I.getSuccessor(0);
2768 const BasicBlock *EHPadBB = I.getSuccessor(1);
2769
2770 const Function *Fn = I.getCalledFunction();
2771
2772 // FIXME: support invoking patchpoint and statepoint intrinsics.
2773 if (Fn && Fn->isIntrinsic())
2774 return false;
2775
2776 // FIXME: support whatever these are.
2777 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2778 return false;
2779
2780 // FIXME: support control flow guard targets.
2781 if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2782 return false;
2783
2784 // FIXME: support Windows exception handling.
2785 if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2786 return false;
2787
2788 // FIXME: support Windows dllimport function calls and calls through
2789 // weak symbols.
2790 if (Fn && (Fn->hasDLLImportStorageClass() ||
2792 Fn->hasExternalWeakLinkage())))
2793 return false;
2794
2795 bool LowerInlineAsm = I.isInlineAsm();
2796 bool NeedEHLabel = true;
2797
2798 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2799 // the region covered by the try.
2800 MCSymbol *BeginSymbol = nullptr;
2801 if (NeedEHLabel) {
2802 MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2803 BeginSymbol = Context.createTempSymbol();
2804 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2805 }
2806
2807 if (LowerInlineAsm) {
2808 if (!translateInlineAsm(I, MIRBuilder))
2809 return false;
2810 } else if (!translateCallBase(I, MIRBuilder))
2811 return false;
2812
2813 MCSymbol *EndSymbol = nullptr;
2814 if (NeedEHLabel) {
2815 EndSymbol = Context.createTempSymbol();
2816 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2817 }
2818
2820 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2821 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2822 BranchProbability EHPadBBProb =
2823 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2825
2826 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2827 return false;
2828
2829 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2830 &ReturnMBB = getMBB(*ReturnBB);
2831 // Update successor info.
2832 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2833 for (auto &UnwindDest : UnwindDests) {
2834 UnwindDest.first->setIsEHPad();
2835 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2836 }
2837 InvokeMBB->normalizeSuccProbs();
2838
2839 if (NeedEHLabel) {
2840 assert(BeginSymbol && "Expected a begin symbol!");
2841 assert(EndSymbol && "Expected an end symbol!");
2842 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2843 }
2844
2845 MIRBuilder.buildBr(ReturnMBB);
2846 return true;
2847}
2848
2849bool IRTranslator::translateCallBr(const User &U,
2850 MachineIRBuilder &MIRBuilder) {
2851 // FIXME: Implement this.
2852 return false;
2853}
2854
2855bool IRTranslator::translateLandingPad(const User &U,
2856 MachineIRBuilder &MIRBuilder) {
2857 const LandingPadInst &LP = cast<LandingPadInst>(U);
2858
2859 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2860
2861 MBB.setIsEHPad();
2862
2863 // If there aren't registers to copy the values into (e.g., during SjLj
2864 // exceptions), then don't bother.
2865 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2866 if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
2867 TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
2868 return true;
2869
2870 // If landingpad's return type is token type, we don't create DAG nodes
2871 // for its exception pointer and selector value. The extraction of exception
2872 // pointer or selector value from token type landingpads is not currently
2873 // supported.
2874 if (LP.getType()->isTokenTy())
2875 return true;
2876
2877 // Add a label to mark the beginning of the landing pad. Deletion of the
2878 // landing pad can thus be detected via the MachineModuleInfo.
2879 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
2880 .addSym(MF->addLandingPad(&MBB));
2881
2882 // If the unwinder does not preserve all registers, ensure that the
2883 // function marks the clobbered registers as used.
2885 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2887
2888 LLT Ty = getLLTForType(*LP.getType(), *DL);
2890 MIRBuilder.buildUndef(Undef);
2891
2893 for (Type *Ty : cast<StructType>(LP.getType())->elements())
2894 Tys.push_back(getLLTForType(*Ty, *DL));
2895 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2896
2897 // Mark exception register as live in.
2898 Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
2899 if (!ExceptionReg)
2900 return false;
2901
2902 MBB.addLiveIn(ExceptionReg);
2903 ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2904 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2905
2906 Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
2907 if (!SelectorReg)
2908 return false;
2909
2910 MBB.addLiveIn(SelectorReg);
2911 Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2912 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2913 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2914
2915 return true;
2916}
2917
2918bool IRTranslator::translateAlloca(const User &U,
2919 MachineIRBuilder &MIRBuilder) {
2920 auto &AI = cast<AllocaInst>(U);
2921
2922 if (AI.isSwiftError())
2923 return true;
2924
2925 if (AI.isStaticAlloca()) {
2926 Register Res = getOrCreateVReg(AI);
2927 int FI = getOrCreateFrameIndex(AI);
2928 MIRBuilder.buildFrameIndex(Res, FI);
2929 return true;
2930 }
2931
2932 // FIXME: support stack probing for Windows.
2934 return false;
2935
2936 // Now we're in the harder dynamic case.
2937 Register NumElts = getOrCreateVReg(*AI.getArraySize());
2938 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2939 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
2940 if (MRI->getType(NumElts) != IntPtrTy) {
2941 Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
2942 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
2943 NumElts = ExtElts;
2944 }
2945
2946 Type *Ty = AI.getAllocatedType();
2947
2948 Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
2949 Register TySize =
2950 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
2951 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
2952
2953 // Round the size of the allocation up to the stack alignment size
2954 // by add SA-1 to the size. This doesn't overflow because we're computing
2955 // an address inside an alloca.
2956 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
2957 auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
2958 auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2960 auto AlignCst =
2961 MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
2962 auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
2963
2964 Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
2965 if (Alignment <= StackAlign)
2966 Alignment = Align(1);
2967 MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
2968
2969 MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
2971 return true;
2972}
2973
2974bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
2975 // FIXME: We may need more info about the type. Because of how LLT works,
2976 // we're completely discarding the i64/double distinction here (amongst
2977 // others). Fortunately the ABIs I know of where that matters don't use va_arg
2978 // anyway but that's not guaranteed.
2979 MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2980 {getOrCreateVReg(*U.getOperand(0)),
2981 DL->getABITypeAlign(U.getType()).value()});
2982 return true;
2983}
2984
2985bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
2987 return true;
2988
2989 auto &UI = cast<UnreachableInst>(U);
2990 // We may be able to ignore unreachable behind a noreturn call.
2992 const BasicBlock &BB = *UI.getParent();
2993 if (&UI != &BB.front()) {
2995 std::prev(BasicBlock::const_iterator(UI));
2996 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2997 if (Call->doesNotReturn())
2998 return true;
2999 }
3000 }
3001 }
3002
3003 MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
3004 return true;
3005}
3006
3007bool IRTranslator::translateInsertElement(const User &U,
3008 MachineIRBuilder &MIRBuilder) {
3009 // If it is a <1 x Ty> vector, use the scalar as it is
3010 // not a legal vector type in LLT.
3011 if (auto *FVT = dyn_cast<FixedVectorType>(U.getType());
3012 FVT && FVT->getNumElements() == 1)
3013 return translateCopy(U, *U.getOperand(1), MIRBuilder);
3014
3015 Register Res = getOrCreateVReg(U);
3016 Register Val = getOrCreateVReg(*U.getOperand(0));
3017 Register Elt = getOrCreateVReg(*U.getOperand(1));
3018 Register Idx = getOrCreateVReg(*U.getOperand(2));
3019 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3020 return true;
3021}
3022
3023bool IRTranslator::translateExtractElement(const User &U,
3024 MachineIRBuilder &MIRBuilder) {
3025 // If it is a <1 x Ty> vector, use the scalar as it is
3026 // not a legal vector type in LLT.
3027 if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
3028 return translateCopy(U, *U.getOperand(0), MIRBuilder);
3029
3030 Register Res = getOrCreateVReg(U);
3031 Register Val = getOrCreateVReg(*U.getOperand(0));
3032 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3033 Register Idx;
3034 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
3035 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3036 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3037 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
3038 Idx = getOrCreateVReg(*NewIdxCI);
3039 }
3040 }
3041 if (!Idx)
3042 Idx = getOrCreateVReg(*U.getOperand(1));
3043 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
3044 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3045 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
3046 }
3047 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3048 return true;
3049}
3050
3051bool IRTranslator::translateShuffleVector(const User &U,
3052 MachineIRBuilder &MIRBuilder) {
3053 // A ShuffleVector that has operates on scalable vectors is a splat vector
3054 // where the value of the splat vector is the 0th element of the first
3055 // operand, since the index mask operand is the zeroinitializer (undef and
3056 // poison are treated as zeroinitializer here).
3057 if (U.getOperand(0)->getType()->isScalableTy()) {
3058 Value *Op0 = U.getOperand(0);
3059 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3061 getOrCreateVReg(*Op0), 0);
3062 MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal);
3063 return true;
3064 }
3065
3067 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
3068 Mask = SVI->getShuffleMask();
3069 else
3070 Mask = cast<ConstantExpr>(U).getShuffleMask();
3071 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3072 MIRBuilder
3073 .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
3074 {getOrCreateVReg(*U.getOperand(0)),
3075 getOrCreateVReg(*U.getOperand(1))})
3076 .addShuffleMask(MaskAlloc);
3077 return true;
3078}
3079
3080bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3081 const PHINode &PI = cast<PHINode>(U);
3082
3084 for (auto Reg : getOrCreateVRegs(PI)) {
3085 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
3086 Insts.push_back(MIB.getInstr());
3087 }
3088
3089 PendingPHIs.emplace_back(&PI, std::move(Insts));
3090 return true;
3091}
3092
3093bool IRTranslator::translateAtomicCmpXchg(const User &U,
3094 MachineIRBuilder &MIRBuilder) {
3095 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
3096
3097 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3098
3099 auto Res = getOrCreateVRegs(I);
3100 Register OldValRes = Res[0];
3101 Register SuccessRes = Res[1];
3102 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3103 Register Cmp = getOrCreateVReg(*I.getCompareOperand());
3104 Register NewVal = getOrCreateVReg(*I.getNewValOperand());
3105
3107 OldValRes, SuccessRes, Addr, Cmp, NewVal,
3109 MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
3110 getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3111 I.getSuccessOrdering(), I.getFailureOrdering()));
3112 return true;
3113}
3114
3115bool IRTranslator::translateAtomicRMW(const User &U,
3116 MachineIRBuilder &MIRBuilder) {
3117 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
3118 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3119
3120 Register Res = getOrCreateVReg(I);
3121 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3122 Register Val = getOrCreateVReg(*I.getValOperand());
3123
3124 unsigned Opcode = 0;
3125 switch (I.getOperation()) {
3126 default:
3127 return false;
3129 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3130 break;
3131 case AtomicRMWInst::Add:
3132 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3133 break;
3134 case AtomicRMWInst::Sub:
3135 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3136 break;
3137 case AtomicRMWInst::And:
3138 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3139 break;
3141 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3142 break;
3143 case AtomicRMWInst::Or:
3144 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3145 break;
3146 case AtomicRMWInst::Xor:
3147 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3148 break;
3149 case AtomicRMWInst::Max:
3150 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3151 break;
3152 case AtomicRMWInst::Min:
3153 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3154 break;
3156 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3157 break;
3159 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3160 break;
3162 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3163 break;
3165 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3166 break;
3168 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3169 break;
3171 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3172 break;
3174 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3175 break;
3177 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3178 break;
3179 }
3180
3181 MIRBuilder.buildAtomicRMW(
3182 Opcode, Res, Addr, Val,
3183 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3184 Flags, MRI->getType(Val), getMemOpAlign(I),
3185 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3186 I.getOrdering()));
3187 return true;
3188}
3189
3190bool IRTranslator::translateFence(const User &U,
3191 MachineIRBuilder &MIRBuilder) {
3192 const FenceInst &Fence = cast<FenceInst>(U);
3193 MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
3194 Fence.getSyncScopeID());
3195 return true;
3196}
3197
3198bool IRTranslator::translateFreeze(const User &U,
3199 MachineIRBuilder &MIRBuilder) {
3200 const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
3201 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
3202
3203 assert(DstRegs.size() == SrcRegs.size() &&
3204 "Freeze with different source and destination type?");
3205
3206 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3207 MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
3208 }
3209
3210 return true;
3211}
3212
3213void IRTranslator::finishPendingPhis() {
3214#ifndef NDEBUG
3215 DILocationVerifier Verifier;
3216 GISelObserverWrapper WrapperObserver(&Verifier);
3217 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3218#endif // ifndef NDEBUG
3219 for (auto &Phi : PendingPHIs) {
3220 const PHINode *PI = Phi.first;
3221 if (PI->getType()->isEmptyTy())
3222 continue;
3223 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3224 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3225 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3226#ifndef NDEBUG
3227 Verifier.setCurrentInst(PI);
3228#endif // ifndef NDEBUG
3229
3231 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3232 auto IRPred = PI->getIncomingBlock(i);
3233 ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
3234 for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
3235 if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
3236 continue;
3237 SeenPreds.insert(Pred);
3238 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3239 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3240 MIB.addUse(ValRegs[j]);
3241 MIB.addMBB(Pred);
3242 }
3243 }
3244 }
3245 }
3246}
3247
3248void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3249 const DILocalVariable *Variable,
3250 const DIExpression *Expression,
3251 const DebugLoc &DL,
3252 MachineIRBuilder &MIRBuilder) {
3253 assert(Variable->isValidLocationForIntrinsic(DL) &&
3254 "Expected inlined-at fields to agree");
3255 // Act as if we're handling a debug intrinsic.
3256 MIRBuilder.setDebugLoc(DL);
3257
3258 if (!V || HasArgList) {
3259 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3260 // terminate any prior location.
3261 MIRBuilder.buildIndirectDbgValue(0, Variable, Expression);
3262 return;
3263 }
3264
3265 if (const auto *CI = dyn_cast<Constant>(V)) {
3266 MIRBuilder.buildConstDbgValue(*CI, Variable, Expression);
3267 return;
3268 }
3269
3270 if (auto *AI = dyn_cast<AllocaInst>(V);
3271 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3272 // If the value is an alloca and the expression starts with a
3273 // dereference, track a stack slot instead of a register, as registers
3274 // may be clobbered.
3275 auto ExprOperands = Expression->getElements();
3276 auto *ExprDerefRemoved =
3277 DIExpression::get(AI->getContext(), ExprOperands.drop_front());
3278 MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), Variable,
3279 ExprDerefRemoved);
3280 return;
3281 }
3282 if (translateIfEntryValueArgument(false, V, Variable, Expression, DL,
3283 MIRBuilder))
3284 return;
3285 for (Register Reg : getOrCreateVRegs(*V)) {
3286 // FIXME: This does not handle register-indirect values at offset 0. The
3287 // direct/indirect thing shouldn't really be handled by something as
3288 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3289 // pretty baked in right now.
3290 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expression);
3291 }
3292 return;
3293}
3294
3295void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3296 const DILocalVariable *Variable,
3297 const DIExpression *Expression,
3298 const DebugLoc &DL,
3299 MachineIRBuilder &MIRBuilder) {
3300 if (!Address || isa<UndefValue>(Address)) {
3301 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3302 return;
3303 }
3304
3305 assert(Variable->isValidLocationForIntrinsic(DL) &&
3306 "Expected inlined-at fields to agree");
3307 auto AI = dyn_cast<AllocaInst>(Address);
3308 if (AI && AI->isStaticAlloca()) {
3309 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3310 // instructions (in fact, they get ignored if they *do* exist).
3311 MF->setVariableDbgInfo(Variable, Expression,
3312 getOrCreateFrameIndex(*AI), DL);
3313 return;
3314 }
3315
3316 if (translateIfEntryValueArgument(true, Address, Variable,
3317 Expression, DL,
3318 MIRBuilder))
3319 return;
3320
3321 // A dbg.declare describes the address of a source variable, so lower it
3322 // into an indirect DBG_VALUE.
3323 MIRBuilder.setDebugLoc(DL);
3324 MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
3325 Variable, Expression);
3326 return;
3327}
3328
3329void IRTranslator::translateDbgInfo(const Instruction &Inst,
3330 MachineIRBuilder &MIRBuilder) {
3331 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3332 if (DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
3333 MIRBuilder.setDebugLoc(DPL->getDebugLoc());
3334 assert(DPL->getLabel() && "Missing label");
3335 assert(DPL->getLabel()->isValidLocationForIntrinsic(
3336 MIRBuilder.getDebugLoc()) &&
3337 "Expected inlined-at fields to agree");
3338 MIRBuilder.buildDbgLabel(DPL->getLabel());
3339 continue;
3340 }
3341 DPValue &DPV = cast<DPValue>(DR);
3342 const DILocalVariable *Variable = DPV.getVariable();
3343 const DIExpression *Expression = DPV.getExpression();
3344 Value *V = DPV.getVariableLocationOp(0);
3345 if (DPV.isDbgDeclare())
3346 translateDbgDeclareRecord(V, DPV.hasArgList(), Variable,
3347 Expression, DPV.getDebugLoc(), MIRBuilder);
3348 else
3349 translateDbgValueRecord(V, DPV.hasArgList(), Variable,
3350 Expression, DPV.getDebugLoc(), MIRBuilder);
3351 }
3352}
3353
3354bool IRTranslator::translate(const Instruction &Inst) {
3355 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3356 CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
3357
3358 if (TLI->fallBackToDAGISel(Inst))
3359 return false;
3360
3361 switch (Inst.getOpcode()) {
3362#define HANDLE_INST(NUM, OPCODE, CLASS) \
3363 case Instruction::OPCODE: \
3364 return translate##OPCODE(Inst, *CurBuilder.get());
3365#include "llvm/IR/Instruction.def"
3366 default:
3367 return false;
3368 }
3369}
3370
3371bool IRTranslator::translate(const Constant &C, Register Reg) {
3372 // We only emit constants into the entry block from here. To prevent jumpy
3373 // debug behaviour remove debug line.
3374 if (auto CurrInstDL = CurBuilder->getDL())
3375 EntryBuilder->setDebugLoc(DebugLoc());
3376
3377 if (auto CI = dyn_cast<ConstantInt>(&C))
3378 EntryBuilder->buildConstant(Reg, *CI);
3379 else if (auto CF = dyn_cast<ConstantFP>(&C))
3380 EntryBuilder->buildFConstant(Reg, *CF);
3381 else if (isa<UndefValue>(C))
3382 EntryBuilder->buildUndef(Reg);
3383 else if (isa<ConstantPointerNull>(C))
3384 EntryBuilder->buildConstant(Reg, 0);
3385 else if (auto GV = dyn_cast<GlobalValue>(&C))
3386 EntryBuilder->buildGlobalValue(Reg, GV);
3387 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
3388 if (!isa<FixedVectorType>(CAZ->getType()))
3389 return false;
3390 // Return the scalar if it is a <1 x Ty> vector.
3391 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3392 if (NumElts == 1)
3393 return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder);
3395 for (unsigned I = 0; I < NumElts; ++I) {
3396 Constant &Elt = *CAZ->getElementValue(I);
3397 Ops.push_back(getOrCreateVReg(Elt));
3398 }
3399 EntryBuilder->buildBuildVector(Reg, Ops);
3400 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
3401 // Return the scalar if it is a <1 x Ty> vector.
3402 if (CV->getNumElements() == 1)
3403 return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
3405 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3406 Constant &Elt = *CV->getElementAsConstant(i);
3407 Ops.push_back(getOrCreateVReg(Elt));
3408 }
3409 EntryBuilder->buildBuildVector(Reg, Ops);
3410 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3411 switch(CE->getOpcode()) {
3412#define HANDLE_INST(NUM, OPCODE, CLASS) \
3413 case Instruction::OPCODE: \
3414 return translate##OPCODE(*CE, *EntryBuilder.get());
3415#include "llvm/IR/Instruction.def"
3416 default:
3417 return false;
3418 }
3419 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3420 if (CV->getNumOperands() == 1)
3421 return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
3423 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3424 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3425 }
3426 EntryBuilder->buildBuildVector(Reg, Ops);
3427 } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3428 EntryBuilder->buildBlockAddress(Reg, BA);
3429 } else
3430 return false;
3431
3432 return true;
3433}
3434
3435bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3437 for (auto &BTB : SL->BitTestCases) {
3438 // Emit header first, if it wasn't already emitted.
3439 if (!BTB.Emitted)
3440 emitBitTestHeader(BTB, BTB.Parent);
3441
3442 BranchProbability UnhandledProb = BTB.Prob;
3443 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3444 UnhandledProb -= BTB.Cases[j].ExtraProb;
3445 // Set the current basic block to the mbb we wish to insert the code into
3446 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3447 // If all cases cover a contiguous range, it is not necessary to jump to
3448 // the default block after the last bit test fails. This is because the
3449 // range check during bit test header creation has guaranteed that every
3450 // case here doesn't go outside the range. In this case, there is no need
3451 // to perform the last bit test, as it will always be true. Instead, make
3452 // the second-to-last bit-test fall through to the target of the last bit
3453 // test, and delete the last bit test.
3454
3455 MachineBasicBlock *NextMBB;
3456 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3457 // Second-to-last bit-test with contiguous range: fall through to the
3458 // target of the final bit test.
3459 NextMBB = BTB.Cases[j + 1].TargetBB;
3460 } else if (j + 1 == ej) {
3461 // For the last bit test, fall through to Default.
3462 NextMBB = BTB.Default;
3463 } else {
3464 // Otherwise, fall through to the next bit test.
3465 NextMBB = BTB.Cases[j + 1].ThisBB;
3466 }
3467
3468 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3469
3470 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3471 // We need to record the replacement phi edge here that normally
3472 // happens in emitBitTestCase before we delete the case, otherwise the
3473 // phi edge will be lost.
3474 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3475 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3476 MBB);
3477 // Since we're not going to use the final bit test, remove it.
3478 BTB.Cases.pop_back();
3479 break;
3480 }
3481 }
3482 // This is "default" BB. We have two jumps to it. From "header" BB and from
3483 // last "case" BB, unless the latter was skipped.
3484 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3485 BTB.Default->getBasicBlock()};
3486 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3487 if (!BTB.ContiguousRange) {
3488 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3489 }
3490 }
3491 SL->BitTestCases.clear();
3492
3493 for (auto &JTCase : SL->JTCases) {
3494 // Emit header first, if it wasn't already emitted.
3495 if (!JTCase.first.Emitted)
3496 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3497
3498 emitJumpTable(JTCase.second, JTCase.second.MBB);
3499 }
3500 SL->JTCases.clear();
3501
3502 for (auto &SwCase : SL->SwitchCases)
3503 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3504 SL->SwitchCases.clear();
3505
3506 // Check if we need to generate stack-protector guard checks.
3507 StackProtector &SP = getAnalysis<StackProtector>();
3508 if (SP.shouldEmitSDCheck(BB)) {
3509 bool FunctionBasedInstrumentation =
3511 SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3512 }
3513 // Handle stack protector.
3514 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3515 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3516 return false;
3517 } else if (SPDescriptor.shouldEmitStackProtector()) {
3518 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3519 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3520
3521 // Find the split point to split the parent mbb. At the same time copy all
3522 // physical registers used in the tail of parent mbb into virtual registers
3523 // before the split point and back into physical registers after the split
3524 // point. This prevents us needing to deal with Live-ins and many other
3525 // register allocation issues caused by us splitting the parent mbb. The
3526 // register allocator will clean up said virtual copies later on.
3528 ParentMBB, *MF->getSubtarget().getInstrInfo());
3529
3530 // Splice the terminator of ParentMBB into SuccessMBB.
3531 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3532 ParentMBB->end());
3533
3534 // Add compare/jump on neq/jump to the parent BB.
3535 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3536 return false;
3537
3538 // CodeGen Failure MBB if we have not codegened it yet.
3539 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3540 if (FailureMBB->empty()) {
3541 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3542 return false;
3543 }
3544
3545 // Clear the Per-BB State.
3546 SPDescriptor.resetPerBBState();
3547 }
3548 return true;
3549}
3550
3551bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3552 MachineBasicBlock *ParentBB) {
3553 CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3554 // First create the loads to the guard/stack slot for the comparison.
3556 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3557 LLT PtrMemTy = getLLTForMVT(TLI->getPointerMemTy(*DL));
3558
3559 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3560 int FI = MFI.getStackProtectorIndex();
3561
3562 Register Guard;
3563 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3564 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3565 Align Align = DL->getPrefTypeAlign(PointerType::getUnqual(M.getContext()));
3566
3567 // Generate code to load the content of the guard slot.
3568 Register GuardVal =
3569 CurBuilder
3570 ->buildLoad(PtrMemTy, StackSlotPtr,
3573 .getReg(0);
3574
3575 if (TLI->useStackGuardXorFP()) {
3576 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3577 return false;
3578 }
3579
3580 // Retrieve guard check function, nullptr if instrumentation is inlined.
3581 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M)) {
3582 // This path is currently untestable on GlobalISel, since the only platform
3583 // that needs this seems to be Windows, and we fall back on that currently.
3584 // The code still lives here in case that changes.
3585 // Silence warning about unused variable until the code below that uses
3586 // 'GuardCheckFn' is enabled.
3587 (void)GuardCheckFn;
3588 return false;
3589#if 0
3590 // The target provides a guard check function to validate the guard value.
3591 // Generate a call to that function with the content of the guard slot as
3592 // argument.
3593 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3594 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3596 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3597 Flags.setInReg();
3598 CallLowering::ArgInfo GuardArgInfo(
3599 {GuardVal, FnTy->getParamType(0), {Flags}});
3600
3602 Info.OrigArgs.push_back(GuardArgInfo);
3603 Info.CallConv = GuardCheckFn->getCallingConv();
3604 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3605 Info.OrigRet = {Register(), FnTy->getReturnType()};
3606 if (!CLI->lowerCall(MIRBuilder, Info)) {
3607 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3608 return false;
3609 }
3610 return true;
3611#endif
3612 }
3613
3614 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3615 // Otherwise, emit a volatile load to retrieve the stack guard value.
3616 if (TLI->useLoadStackGuardNode()) {
3617 Guard =
3619 getStackGuard(Guard, *CurBuilder);
3620 } else {
3621 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3622 const Value *IRGuard = TLI->getSDagStackGuard(M);
3623 Register GuardPtr = getOrCreateVReg(*IRGuard);
3624
3625 Guard = CurBuilder
3626 ->buildLoad(PtrMemTy, GuardPtr,
3630 .getReg(0);
3631 }
3632
3633 // Perform the comparison.
3634 auto Cmp =
3635 CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3636 // If the guard/stackslot do not equal, branch to failure MBB.
3637 CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3638 // Otherwise branch to success MBB.
3639 CurBuilder->buildBr(*SPD.getSuccessMBB());
3640 return true;
3641}
3642
3643bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3644 MachineBasicBlock *FailureBB) {
3645 CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3646
3647 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3648 const char *Name = TLI->getLibcallName(Libcall);
3649
3651 Info.CallConv = TLI->getLibcallCallingConv(Libcall);
3653 Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3654 0};
3655 if (!CLI->lowerCall(*CurBuilder, Info)) {
3656 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3657 return false;
3658 }
3659
3660 // On PS4/PS5, the "return address" must still be within the calling
3661 // function, even if it's at the very end, so emit an explicit TRAP here.
3662 // WebAssembly needs an unreachable instruction after a non-returning call,
3663 // because the function return type can be different from __stack_chk_fail's
3664 // return type (void).
3665 const TargetMachine &TM = MF->getTarget();
3666 if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3667 LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3668 return false;
3669 }
3670 return true;
3671}
3672
3673void IRTranslator::finalizeFunction() {
3674 // Release the memory used by the different maps we
3675 // needed during the translation.
3676 PendingPHIs.clear();
3677 VMap.reset();
3678 FrameIndices.clear();
3679 MachinePreds.clear();
3680 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3681 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3682 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3683 EntryBuilder.reset();
3684 CurBuilder.reset();
3685 FuncInfo.clear();
3686 SPDescriptor.resetPerFunctionState();
3687}
3688
3689/// Returns true if a BasicBlock \p BB within a variadic function contains a
3690/// variadic musttail call.
3691static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3692 if (!IsVarArg)
3693 return false;
3694
3695 // Walk the block backwards, because tail calls usually only appear at the end
3696 // of a block.
3697 return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3698 const auto *CI = dyn_cast<CallInst>(&I);
3699 return CI && CI->isMustTailCall();
3700 });
3701}
3702
3704 MF = &CurMF;
3705 const Function &F = MF->getFunction();
3707 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3708 // Set the CSEConfig and run the analysis.
3709 GISelCSEInfo *CSEInfo = nullptr;
3710 TPC = &getAnalysis<TargetPassConfig>();
3711 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3713 : TPC->isGISelCSEEnabled();
3714 TLI = MF->getSubtarget().getTargetLowering();
3715
3716 if (EnableCSE) {
3717 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3718 CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3719 EntryBuilder->setCSEInfo(CSEInfo);
3720 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3721 CurBuilder->setCSEInfo(CSEInfo);
3722 } else {
3723 EntryBuilder = std::make_unique<MachineIRBuilder>();
3724 CurBuilder = std::make_unique<MachineIRBuilder>();
3725 }
3726 CLI = MF->getSubtarget().getCallLowering();
3727 CurBuilder->setMF(*MF);
3728 EntryBuilder->setMF(*MF);
3729 MRI = &MF->getRegInfo();
3730 DL = &F.getParent()->getDataLayout();
3731 ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3732 const TargetMachine &TM = MF->getTarget();
3733 TM.resetTargetOptions(F);
3734 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3735 FuncInfo.MF = MF;
3736 if (EnableOpts) {
3737 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3738 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3739 } else {
3740 AA = nullptr;
3741 FuncInfo.BPI = nullptr;
3742 }
3743
3744 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3745 MF->getFunction());
3746 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3747 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3748
3749 SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3750 SL->init(*TLI, TM, *DL);
3751
3752 assert(PendingPHIs.empty() && "stale PHIs");
3753
3754 // Targets which want to use big endian can enable it using
3755 // enableBigEndian()
3756 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3757 // Currently we don't properly handle big endian code.
3758 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3759 F.getSubprogram(), &F.getEntryBlock());
3760 R << "unable to translate in big endian mode";
3761 reportTranslationError(*MF, *TPC, *ORE, R);
3762 }
3763
3764 // Release the per-function state when we return, whether we succeeded or not.
3765 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3766
3767 // Setup a separate basic-block for the arguments and constants
3769 MF->push_back(EntryBB);
3770 EntryBuilder->setMBB(*EntryBB);
3771
3772 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3773 SwiftError.setFunction(CurMF);
3774 SwiftError.createEntriesInEntryBlock(DbgLoc);
3775
3776 bool IsVarArg = F.isVarArg();
3777 bool HasMustTailInVarArgFn = false;
3778
3779 // Create all blocks, in IR order, to preserve the layout.
3780 for (const BasicBlock &BB: F) {
3781 auto *&MBB = BBToMBB[&BB];
3782
3783 MBB = MF->CreateMachineBasicBlock(&BB);
3784 MF->push_back(MBB);
3785
3786 if (BB.hasAddressTaken())
3787 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3788
3789 if (!HasMustTailInVarArgFn)
3790 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3791 }
3792
3793 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3794
3795 // Make our arguments/constants entry block fallthrough to the IR entry block.
3796 EntryBB->addSuccessor(&getMBB(F.front()));
3797
3798 if (CLI->fallBackToDAGISel(*MF)) {
3799 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3800 F.getSubprogram(), &F.getEntryBlock());
3801 R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3802 reportTranslationError(*MF, *TPC, *ORE, R);
3803 return false;
3804 }
3805
3806 // Lower the actual args into this basic block.
3807 SmallVector<ArrayRef<Register>, 8> VRegArgs;
3808 for (const Argument &Arg: F.args()) {
3809 if (DL->getTypeStoreSize(Arg.getType()).isZero())
3810 continue; // Don't handle zero sized types.
3811 ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3812 VRegArgs.push_back(VRegs);
3813
3814 if (Arg.hasSwiftErrorAttr()) {
3815 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3816 SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3817 }
3818 }
3819
3820 if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
3821 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3822 F.getSubprogram(), &F.getEntryBlock());
3823 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3824 reportTranslationError(*MF, *TPC, *ORE, R);
3825 return false;
3826 }
3827
3828 // Need to visit defs before uses when translating instructions.
3829 GISelObserverWrapper WrapperObserver;
3830 if (EnableCSE && CSEInfo)
3831 WrapperObserver.addObserver(CSEInfo);
3832 {
3834#ifndef NDEBUG
3835 DILocationVerifier Verifier;
3836 WrapperObserver.addObserver(&Verifier);
3837#endif // ifndef NDEBUG
3838 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3839 RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3840 for (const BasicBlock *BB : RPOT) {
3841 MachineBasicBlock &MBB = getMBB(*BB);
3842 // Set the insertion point of all the following translations to
3843 // the end of this basic block.
3844 CurBuilder->setMBB(MBB);
3845 HasTailCall = false;
3846 for (const Instruction &Inst : *BB) {
3847 // If we translated a tail call in the last step, then we know
3848 // everything after the call is either a return, or something that is
3849 // handled by the call itself. (E.g. a lifetime marker or assume
3850 // intrinsic.) In this case, we should stop translating the block and
3851 // move on.
3852 if (HasTailCall)
3853 break;
3854#ifndef NDEBUG
3855 Verifier.setCurrentInst(&Inst);
3856#endif // ifndef NDEBUG
3857
3858 // Translate any debug-info attached to the instruction.
3859 translateDbgInfo(Inst, *CurBuilder.get());
3860
3861 if (translate(Inst))
3862 continue;
3863
3864 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3865 Inst.getDebugLoc(), BB);
3866 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3867
3868 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3869 std::string InstStrStorage;
3870 raw_string_ostream InstStr(InstStrStorage);
3871 InstStr << Inst;
3872
3873 R << ": '" << InstStr.str() << "'";
3874 }
3875
3876 reportTranslationError(*MF, *TPC, *ORE, R);
3877 return false;
3878 }
3879
3880 if (!finalizeBasicBlock(*BB, MBB)) {
3881 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3882 BB->getTerminator()->getDebugLoc(), BB);
3883 R << "unable to translate basic block";
3884 reportTranslationError(*MF, *TPC, *ORE, R);
3885 return false;
3886 }
3887 }
3888#ifndef NDEBUG
3889 WrapperObserver.removeObserver(&Verifier);
3890#endif
3891 }
3892
3893 finishPendingPhis();
3894
3895 SwiftError.propagateVRegs();
3896
3897 // Merge the argument lowering and constants block with its single
3898 // successor, the LLVM-IR entry block. We want the basic block to
3899 // be maximal.
3900 assert(EntryBB->succ_size() == 1 &&
3901 "Custom BB used for lowering should have only one successor");
3902 // Get the successor of the current entry block.
3903 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3904 assert(NewEntryBB.pred_size() == 1 &&
3905 "LLVM-IR entry block has a predecessor!?");
3906 // Move all the instruction from the current entry block to the
3907 // new entry block.
3908 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3909 EntryBB->end());
3910
3911 // Update the live-in information for the new entry block.
3912 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3913 NewEntryBB.addLiveIn(LiveIn);
3914 NewEntryBB.sortUniqueLiveIns();
3915
3916 // Get rid of the now empty basic block.
3917 EntryBB->removeSuccessor(&NewEntryBB);
3918 MF->remove(EntryBB);
3919 MF->deleteMachineBasicBlock(EntryBB);
3920
3921 assert(&MF->front() == &NewEntryBB &&
3922 "New entry wasn't next in the list of basic block!");
3923
3924 // Initialize stack protector information.
3925 StackProtector &SP = getAnalysis<StackProtector>();
3927
3928 return false;
3929}
unsigned SubReg
#define Success
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:301
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
IRTranslator LLVM IR MI
#define DEBUG_TYPE
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
uint64_t High
IntegerType * Int32Ty
LLVMContext & Context
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Value * RHS
Value * LHS
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
an instruction to allocate memory on the stack
Definition: Instructions.h:59
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:157
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:107
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:103
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:842
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:349
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:639
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:165
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:347
const Instruction & front() const
Definition: BasicBlock.h:452
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:366
const Instruction & back() const
Definition: BasicBlock.h:454
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1770
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1703
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1623
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2333
Value * getCalledOperand() const
Definition: InstrTypes.h:1696
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1629
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1639
unsigned arg_size() const
Definition: InstrTypes.h:1646
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1780
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:543
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:591
virtual bool supportSwiftError() const
Definition: CallLowering.h:446
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:511
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:555
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:529
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:982
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:994
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:995
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:988
@ ICMP_EQ
equal
Definition: InstrTypes.h:986
@ ICMP_NE
not equal
Definition: InstrTypes.h:987
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:991
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:967
bool isFPPredicate() const
Definition: InstrTypes.h:1083
bool isIntPredicate() const
Definition: InstrTypes.h:1084
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:204
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:410
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:905
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:672
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:742
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
Definition: Instructions.h:460
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:498
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:487
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1828
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1874
const Function & getFunction() const
Definition: Function.h:160
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:342
The actual analysis pass wrapper.
Definition: CSEInfo.h:222
Simple wrapper that does the following.
Definition: CSEInfo.h:204
The CSE Analysis object.
Definition: CSEInfo.h:69
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:528
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
static char ID
Definition: IRTranslator.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgRecordRange() const
Return a range over the DbgRecords attached to this instruction.
Definition: Instruction.h:83
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:340
const BasicBlock * getParent() const
Definition: Instruction.h:151
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:358
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1704
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:251
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelType.h:214
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelType.h:159
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
constexpr bool isPointer() const
Definition: LowLevelType.h:149
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:100
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
Definition: LowLevelType.h:178
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
Value * getPointerOperand()
Definition: Instructions.h:280
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:245
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:255
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, unsigned Locality, unsigned CacheType, MachineMemOperand &MMO)
Build and insert G_PREFETCH Addr, RW, Locality, CacheType.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val)
Build and insert Res = G_SPLAT_VECTOR Val.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:69
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:554
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MCContext & getContext() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
Diagnostic information for missed-optimization remarks.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A simple RAII based Delegate installer.
A simple RAII based Observer installer.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition: Register.h:110
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
bool shouldEmitFunctionBasedCheckStackProtector() const
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool fallBackToDAGISel(const Instruction &Inst) const
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
TargetOptions Options
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Target-Independent Code Generator Pass Configuration Options.
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const InlineAsmLowering * getInlineAsmLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const CallLowering * getCallLowering() const
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:608
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:302
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:295
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:693
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
constexpr bool isZero() const
Definition: TypeSize.h:156
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:316
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
@ CE
Windows NT (Windows on ARM)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
@ Offset
Definition: DWP.cpp:456
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2415
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
gep_type_iterator gep_type_end(const User *GEP)
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:241
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:141
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
@ Global
Append to llvm.global_dtors.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:1071
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1963
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ FMul
Product of floats.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1833
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
unsigned succ_size(const MachineBasicBlock *BB)
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Pair of physical register and lane mask.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool canHandle(const Instruction *I, const TargetLibraryInfo &TLI)
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
struct PredInfoPair PredInfo