48#include "llvm/IR/IntrinsicsAMDGPU.h"
102static SGPRRegisterRegAlloc
103defaultSGPRRegAlloc(
"default",
104 "pick SGPR register allocator based on -O option",
107static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor,
false,
110 cl::desc(
"Register allocator to use for SGPRs"));
112static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor,
false,
115 cl::desc(
"Register allocator to use for VGPRs"));
118static void initializeDefaultSGPRRegisterAllocatorOnce() {
123 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
127static void initializeDefaultVGPRRegisterAllocatorOnce() {
132 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
136static FunctionPass *createBasicSGPRRegisterAllocator() {
140static FunctionPass *createGreedySGPRRegisterAllocator() {
148static FunctionPass *createBasicVGPRRegisterAllocator() {
152static FunctionPass *createGreedyVGPRRegisterAllocator() {
160static SGPRRegisterRegAlloc basicRegAllocSGPR(
161 "basic",
"basic register allocator", createBasicSGPRRegisterAllocator);
162static SGPRRegisterRegAlloc greedyRegAllocSGPR(
163 "greedy",
"greedy register allocator", createGreedySGPRRegisterAllocator);
165static SGPRRegisterRegAlloc fastRegAllocSGPR(
166 "fast",
"fast register allocator", createFastSGPRRegisterAllocator);
169static VGPRRegisterRegAlloc basicRegAllocVGPR(
170 "basic",
"basic register allocator", createBasicVGPRRegisterAllocator);
171static VGPRRegisterRegAlloc greedyRegAllocVGPR(
172 "greedy",
"greedy register allocator", createGreedyVGPRRegisterAllocator);
174static VGPRRegisterRegAlloc fastRegAllocVGPR(
175 "fast",
"fast register allocator", createFastVGPRRegisterAllocator);
180 cl::desc(
"Run early if-conversion"),
185 cl::desc(
"Run pre-RA exec mask optimizations"),
190 cl::desc(
"Lower GPU ctor / dtors to globals on the device."),
195 "amdgpu-load-store-vectorizer",
196 cl::desc(
"Enable load store vectorizer"),
202 "amdgpu-scalarize-global-loads",
203 cl::desc(
"Enable global load scalarization"),
209 "amdgpu-internalize-symbols",
210 cl::desc(
"Enable elimination of non-kernel functions and unused globals"),
216 "amdgpu-early-inline-all",
217 cl::desc(
"Inline all functions early"),
222 "amdgpu-enable-remove-incompatible-functions",
cl::Hidden,
223 cl::desc(
"Enable removal of functions when they"
224 "use features not supported by the target GPU"),
228 "amdgpu-sdwa-peephole",
233 "amdgpu-dpp-combine",
239 cl::desc(
"Enable AMDGPU Alias Analysis"),
244 "amdgpu-late-structurize",
245 cl::desc(
"Enable late CFG structurization"),
252 "amdgpu-disable-structurizer",
253 cl::desc(
"Disable structurizer for experiments; produces unusable code"),
258 "amdgpu-simplify-libcall",
259 cl::desc(
"Enable amdgpu library simplifications"),
264 "amdgpu-ir-lower-kernel-arguments",
265 cl::desc(
"Lower kernel argument loads in IR pass"),
270 "amdgpu-reassign-regs",
271 cl::desc(
"Enable register reassign optimizations on gfx10+"),
276 "amdgpu-opt-vgpr-liverange",
277 cl::desc(
"Enable VGPR liverange optimizations for if-else structure"),
281 "amdgpu-atomic-optimizer-strategy",
282 cl::desc(
"Select DPP or Iterative strategy for scan"),
285 clEnumValN(ScanOptions::DPP,
"DPP",
"Use DPP operations for scan"),
286 clEnumValN(ScanOptions::Iterative,
"Iterative",
287 "Use Iterative approach for scan"),
288 clEnumValN(ScanOptions::None,
"None",
"Disable atomic optimizer")));
292 "amdgpu-mode-register",
293 cl::desc(
"Enable mode register pass"),
300 cl::desc(
"Enable s_singleuse_vdst insertion"),
306 cl::desc(
"Enable s_delay_alu insertion"),
312 cl::desc(
"Enable VOPD, dual issue of VALU in wave32"),
319 cl::desc(
"Enable machine DCE inside regalloc"));
326 "amdgpu-scalar-ir-passes",
327 cl::desc(
"Enable scalar IR passes"),
332 "amdgpu-enable-structurizer-workarounds",
337 "amdgpu-enable-lower-module-lds",
cl::desc(
"Enable lower module lds pass"),
342 "amdgpu-enable-pre-ra-optimizations",
347 "amdgpu-enable-promote-kernel-arguments",
348 cl::desc(
"Enable promotion of flat kernel pointer arguments to global"),
352 "amdgpu-enable-image-intrinsic-optimizer",
358 cl::desc(
"Enable loop data prefetch on AMDGPU"),
362 "amdgpu-enable-max-ilp-scheduling-strategy",
363 cl::desc(
"Enable scheduling strategy to maximize ILP for a single wave."),
367 "amdgpu-enable-rewrite-partial-reg-uses",
372 "amdgpu-enable-hipstdpar",
373 cl::desc(
"Enable HIP Standard Parallelism Offload support"),
cl::init(
false),
458 return std::make_unique<AMDGPUTargetObjectFile>();
471 if (ST.shouldClusterStores())
493 if (ST.shouldClusterStores())
509 if (ST.shouldClusterStores())
521 "Run GCN scheduler to maximize occupancy",
529 "gcn-iterative-max-occupancy-experimental",
530 "Run GCN scheduler to maximize occupancy (experimental)",
534 "gcn-iterative-minreg",
535 "Run GCN iterative scheduler for minimal register usage (experimental)",
540 "Run GCN iterative scheduler for ILP scheduling (experimental)",
546 return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
547 "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1";
556 return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
557 "-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-"
559 "128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-"
584 std::optional<Reloc::Model> RM,
585 std::optional<CodeModel::Model> CM,
608 Attribute GPUAttr =
F.getFnAttribute(
"target-cpu");
613 Attribute FSAttr =
F.getFnAttribute(
"target-features");
621 if (
const Function *
F = dyn_cast<Function>(&GV))
622 return F->isDeclaration() ||
F->getName().starts_with(
"__asan_") ||
623 F->getName().starts_with(
"__sanitizer_") ||
652#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
742 unsigned DestAS)
const {
748 const auto *LD = dyn_cast<LoadInst>(V);
753 assert(V->getType()->isPointerTy() &&
756 const auto *
Ptr = LD->getPointerOperand();
766std::pair<const Value *, unsigned>
768 if (
auto *II = dyn_cast<IntrinsicInst>(V)) {
769 switch (II->getIntrinsicID()) {
770 case Intrinsic::amdgcn_is_shared:
772 case Intrinsic::amdgcn_is_private:
777 return std::pair(
nullptr, -1);
784 const_cast<Value *
>(V),
786 m_Not(m_Intrinsic<Intrinsic::amdgcn_is_private>(
790 return std::pair(
nullptr, -1);
816 std::optional<Reloc::Model> RM,
817 std::optional<CodeModel::Model> CM,
829 auto &
I = SubtargetMap[SubtargetKey];
835 I = std::make_unique<GCNSubtarget>(
TargetTriple, GPU, FS, *
this);
865 setRequiresCodeGenSCCOrder(
true);
870 return getTM<GCNTargetMachine>();
879 C, std::make_unique<PostGenericScheduler>(
C),
883 if (
ST.shouldClusterStores())
888 if (isPassEnabled(
EnableVOPD, CodeGenOptLevel::Less))
893 bool addPreISel()
override;
894 void addMachineSSAOptimization()
override;
895 bool addILPOpts()
override;
896 bool addInstSelector()
override;
897 bool addIRTranslator()
override;
898 void addPreLegalizeMachineIR()
override;
899 bool addLegalizeMachineIR()
override;
900 void addPreRegBankSelect()
override;
901 bool addRegBankSelect()
override;
902 void addPreGlobalInstructionSelect()
override;
903 bool addGlobalInstructionSelect()
override;
904 void addFastRegAlloc()
override;
905 void addOptimizedRegAlloc()
override;
909 FunctionPass *createRegAllocPass(
bool Optimized)
override;
911 bool addRegAssignAndRewriteFast()
override;
912 bool addRegAssignAndRewriteOptimized()
override;
914 void addPreRegAlloc()
override;
915 bool addPreRewrite()
override;
916 void addPostRegAlloc()
override;
917 void addPreSched2()
override;
918 void addPreEmitPass()
override;
1121 if (ST.shouldClusterStores())
1129 return R600MachineFunctionInfo::create<R600MachineFunctionInfo>(
1140 if (ST.enableSIScheduler())
1149bool GCNPassConfig::addPreISel() {
1184void GCNPassConfig::addMachineSSAOptimization() {
1208bool GCNPassConfig::addILPOpts() {
1216bool GCNPassConfig::addInstSelector() {
1223bool GCNPassConfig::addIRTranslator() {
1228void GCNPassConfig::addPreLegalizeMachineIR() {
1234bool GCNPassConfig::addLegalizeMachineIR() {
1239void GCNPassConfig::addPreRegBankSelect() {
1245bool GCNPassConfig::addRegBankSelect() {
1250void GCNPassConfig::addPreGlobalInstructionSelect() {
1255bool GCNPassConfig::addGlobalInstructionSelect() {
1260void GCNPassConfig::addPreRegAlloc() {
1266void GCNPassConfig::addFastRegAlloc() {
1280void GCNPassConfig::addOptimizedRegAlloc() {
1316bool GCNPassConfig::addPreRewrite() {
1323FunctionPass *GCNPassConfig::createSGPRAllocPass(
bool Optimized) {
1326 initializeDefaultSGPRRegisterAllocatorOnce);
1338FunctionPass *GCNPassConfig::createVGPRAllocPass(
bool Optimized) {
1341 initializeDefaultVGPRRegisterAllocatorOnce);
1348 return createGreedyVGPRRegisterAllocator();
1350 return createFastVGPRRegisterAllocator();
1353FunctionPass *GCNPassConfig::createRegAllocPass(
bool Optimized) {
1358 "-regalloc not supported with amdgcn. Use -sgpr-regalloc and -vgpr-regalloc";
1360bool GCNPassConfig::addRegAssignAndRewriteFast() {
1361 if (!usingDefaultRegAlloc())
1366 addPass(createSGPRAllocPass(
false));
1372 addPass(createVGPRAllocPass(
false));
1378bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1379 if (!usingDefaultRegAlloc())
1384 addPass(createSGPRAllocPass(
true));
1396 addPass(createVGPRAllocPass(
true));
1406void GCNPassConfig::addPostRegAlloc() {
1413void GCNPassConfig::addPreSched2() {
1419void GCNPassConfig::addPreEmitPass() {
1455 return new GCNPassConfig(*
this, PM);
1467 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
1494 if (MFI->Occupancy == 0) {
1502 SourceRange =
RegName.SourceRange;
1515 if (parseOptionalRegister(YamlMFI.
VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
1518 if (parseOptionalRegister(YamlMFI.
SGPRForEXECCopy, MFI->SGPRForEXECCopy))
1522 MFI->LongBranchReservedReg))
1531 "incorrect register class for field",
RegName.Value,
1532 std::nullopt, std::nullopt);
1533 SourceRange =
RegName.SourceRange;
1537 if (parseRegister(YamlMFI.
ScratchRSrcReg, MFI->ScratchRSrcReg) ||
1542 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
1543 !AMDGPU::SGPR_128RegClass.contains(MFI->ScratchRSrcReg)) {
1547 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
1548 !AMDGPU::SGPR_32RegClass.contains(MFI->FrameOffsetReg)) {
1552 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
1553 !AMDGPU::SGPR_32RegClass.contains(MFI->StackPtrOffsetReg)) {
1559 if (parseRegister(YamlReg, ParsedReg))
1565 auto parseAndCheckArgument = [&](
const std::optional<yaml::SIArgument> &
A,
1568 unsigned SystemSGPRs) {
1573 if (
A->IsRegister) {
1576 SourceRange =
A->RegisterName.SourceRange;
1579 if (!RC.contains(Reg))
1580 return diagnoseRegisterClass(
A->RegisterName);
1588 MFI->NumUserSGPRs += UserSGPRs;
1589 MFI->NumSystemSGPRs += SystemSGPRs;
1594 (parseAndCheckArgument(YamlMFI.
ArgInfo->PrivateSegmentBuffer,
1595 AMDGPU::SGPR_128RegClass,
1597 parseAndCheckArgument(YamlMFI.
ArgInfo->DispatchPtr,
1598 AMDGPU::SReg_64RegClass, MFI->ArgInfo.
DispatchPtr,
1600 parseAndCheckArgument(YamlMFI.
ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
1602 parseAndCheckArgument(YamlMFI.
ArgInfo->KernargSegmentPtr,
1603 AMDGPU::SReg_64RegClass,
1605 parseAndCheckArgument(YamlMFI.
ArgInfo->DispatchID,
1606 AMDGPU::SReg_64RegClass, MFI->ArgInfo.
DispatchID,
1608 parseAndCheckArgument(YamlMFI.
ArgInfo->FlatScratchInit,
1609 AMDGPU::SReg_64RegClass,
1611 parseAndCheckArgument(YamlMFI.
ArgInfo->PrivateSegmentSize,
1612 AMDGPU::SGPR_32RegClass,
1614 parseAndCheckArgument(YamlMFI.
ArgInfo->LDSKernelId,
1615 AMDGPU::SGPR_32RegClass,
1617 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupIDX,
1620 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupIDY,
1623 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupIDZ,
1626 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupInfo,
1627 AMDGPU::SGPR_32RegClass,
1629 parseAndCheckArgument(YamlMFI.
ArgInfo->PrivateSegmentWaveByteOffset,
1630 AMDGPU::SGPR_32RegClass,
1632 parseAndCheckArgument(YamlMFI.
ArgInfo->ImplicitArgPtr,
1633 AMDGPU::SReg_64RegClass,
1635 parseAndCheckArgument(YamlMFI.
ArgInfo->ImplicitBufferPtr,
1636 AMDGPU::SReg_64RegClass,
1638 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkItemIDX,
1639 AMDGPU::VGPR_32RegClass,
1641 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkItemIDY,
1642 AMDGPU::VGPR_32RegClass,
1644 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkItemIDZ,
1645 AMDGPU::VGPR_32RegClass,
1649 if (ST.hasIEEEMode())
1651 if (ST.hasDX10ClampMode())
static cl::opt< bool > EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, cl::desc("Run early if-conversion"), cl::init(true))
This is the AMGPU address space based alias analysis pass.
static cl::opt< bool > EnableDCEInRA("amdgpu-dce-in-ra", cl::init(true), cl::Hidden, cl::desc("Enable machine DCE inside regalloc"))
static cl::opt< bool, true > EnableLowerModuleLDS("amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"), cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true), cl::Hidden)
static MachineSchedRegistry SISchedRegistry("si", "Run SI's custom scheduler", createSIMachineScheduler)
static ScheduleDAGInstrs * createIterativeILPMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EarlyInlineAll("amdgpu-early-inline-all", cl::desc("Inline all functions early"), cl::init(false), cl::Hidden)
static cl::opt< bool > EnableLowerKernelArguments("amdgpu-ir-lower-kernel-arguments", cl::desc("Lower kernel argument loads in IR pass"), cl::init(true), cl::Hidden)
static ScheduleDAGInstrs * createGCNMaxILPMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EnableSDWAPeephole("amdgpu-sdwa-peephole", cl::desc("Enable SDWA peepholer"), cl::init(true))
static MachineSchedRegistry GCNMinRegSchedRegistry("gcn-iterative-minreg", "Run GCN iterative scheduler for minimal register usage (experimental)", createMinRegScheduler)
static cl::opt< bool > EnableImageIntrinsicOptimizer("amdgpu-enable-image-intrinsic-optimizer", cl::desc("Enable image intrinsic optimizer pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableSIModeRegisterPass("amdgpu-mode-register", cl::desc("Enable mode register pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableDPPCombine("amdgpu-dpp-combine", cl::desc("Enable DPP combiner"), cl::init(true))
static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry("gcn-iterative-max-occupancy-experimental", "Run GCN scheduler to maximize occupancy (experimental)", createIterativeGCNMaxOccupancyMachineScheduler)
static cl::opt< bool > EnableSetWavePriority("amdgpu-set-wave-priority", cl::desc("Adjust wave priority"), cl::init(false), cl::Hidden)
static cl::opt< bool > LowerCtorDtor("amdgpu-lower-global-ctor-dtor", cl::desc("Lower GPU ctor / dtors to globals on the device."), cl::init(true), cl::Hidden)
static cl::opt< bool, true > DisableStructurizer("amdgpu-disable-structurizer", cl::desc("Disable structurizer for experiments; produces unusable code"), cl::location(AMDGPUTargetMachine::DisableStructurizer), cl::ReallyHidden)
static cl::opt< bool > OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden, cl::desc("Run pre-RA exec mask optimizations"), cl::init(true))
static cl::opt< bool > EnablePromoteKernelArguments("amdgpu-enable-promote-kernel-arguments", cl::desc("Enable promotion of flat kernel pointer arguments to global"), cl::Hidden, cl::init(true))
static cl::opt< bool > EnableRewritePartialRegUses("amdgpu-enable-rewrite-partial-reg-uses", cl::desc("Enable rewrite partial reg uses pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableLibCallSimplify("amdgpu-simplify-libcall", cl::desc("Enable amdgpu library simplifications"), cl::init(true), cl::Hidden)
static MachineSchedRegistry GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp", createGCNMaxILPMachineScheduler)
static cl::opt< bool > InternalizeSymbols("amdgpu-internalize-symbols", cl::desc("Enable elimination of non-kernel functions and unused globals"), cl::init(false), cl::Hidden)
static LLVM_READNONE StringRef getGPUOrDefault(const Triple &TT, StringRef GPU)
static Reloc::Model getEffectiveRelocModel(std::optional< Reloc::Model > RM)
static cl::opt< bool > EnableStructurizerWorkarounds("amdgpu-enable-structurizer-workarounds", cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden, cl::desc("Enable AMDGPU Alias Analysis"), cl::init(true))
static Expected< ScanOptions > parseAMDGPUAtomicOptimizerStrategy(StringRef Params)
static ScheduleDAGInstrs * createMinRegScheduler(MachineSchedContext *C)
static cl::opt< bool, true > LateCFGStructurize("amdgpu-late-structurize", cl::desc("Enable late CFG structurization"), cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG), cl::Hidden)
static cl::opt< bool > EnableHipStdPar("amdgpu-enable-hipstdpar", cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false), cl::Hidden)
static cl::opt< bool > EnableInsertDelayAlu("amdgpu-enable-delay-alu", cl::desc("Enable s_delay_alu insertion"), cl::init(true), cl::Hidden)
static ScheduleDAGInstrs * createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EnableLoadStoreVectorizer("amdgpu-load-store-vectorizer", cl::desc("Enable load store vectorizer"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableMaxIlpSchedStrategy("amdgpu-enable-max-ilp-scheduling-strategy", cl::desc("Enable scheduling strategy to maximize ILP for a single wave."), cl::Hidden, cl::init(false))
static bool mustPreserveGV(const GlobalValue &GV)
Predicate for Internalize pass.
static cl::opt< bool > EnableLoopPrefetch("amdgpu-loop-prefetch", cl::desc("Enable loop data prefetch on AMDGPU"), cl::Hidden, cl::init(false))
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget()
static cl::opt< bool > EnableInsertSingleUseVDST("amdgpu-enable-single-use-vdst", cl::desc("Enable s_singleuse_vdst insertion"), cl::init(false), cl::Hidden)
static cl::opt< bool > RemoveIncompatibleFunctions("amdgpu-enable-remove-incompatible-functions", cl::Hidden, cl::desc("Enable removal of functions when they" "use features not supported by the target GPU"), cl::init(true))
static cl::opt< bool > EnableScalarIRPasses("amdgpu-scalar-ir-passes", cl::desc("Enable scalar IR passes"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableRegReassign("amdgpu-reassign-regs", cl::desc("Enable register reassign optimizations on gfx10+"), cl::init(true), cl::Hidden)
static cl::opt< bool > OptVGPRLiveRange("amdgpu-opt-vgpr-liverange", cl::desc("Enable VGPR liverange optimizations for if-else structure"), cl::init(true), cl::Hidden)
static ScheduleDAGInstrs * createSIMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EnablePreRAOptimizations("amdgpu-enable-pre-ra-optimizations", cl::desc("Enable Pre-RA optimizations pass"), cl::init(true), cl::Hidden)
static cl::opt< ScanOptions > AMDGPUAtomicOptimizerStrategy("amdgpu-atomic-optimizer-strategy", cl::desc("Select DPP or Iterative strategy for scan"), cl::init(ScanOptions::Iterative), cl::values(clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"), clEnumValN(ScanOptions::Iterative, "Iterative", "Use Iterative approach for scan"), clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")))
static cl::opt< bool > EnableVOPD("amdgpu-enable-vopd", cl::desc("Enable VOPD, dual issue of VALU in wave32"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden, cl::desc("Run early if-conversion"), cl::init(false))
static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C)
static MachineSchedRegistry GCNILPSchedRegistry("gcn-iterative-ilp", "Run GCN iterative scheduler for ILP scheduling (experimental)", createIterativeILPMachineScheduler)
static cl::opt< bool > ScalarizeGlobal("amdgpu-scalarize-global-loads", cl::desc("Enable global load scalarization"), cl::init(true), cl::Hidden)
static const char RegAllocOptNotSupportedMessage[]
static MachineSchedRegistry GCNMaxOccupancySchedRegistry("gcn-max-occupancy", "Run GCN scheduler to maximize occupancy", createGCNMaxOccupancyMachineScheduler)
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file declares the AMDGPU-specific subclass of TargetLoweringObjectFile.
Provides passes to inlining "always_inline" functions.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This header provides classes for managing passes over SCCs of the call graph.
Provides analysis for continuously CSEing during GISel passes.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
#define LLVM_EXTERNAL_VISIBILITY
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines the class GCNIterativeScheduler, which uses an iterative approach to find a best sc...
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
AcceleratorCodeSelection - Identify all functions reachable from a kernel, removing those that are un...
This file declares the IRTranslator pass.
static std::string computeDataLayout()
unsigned const TargetRegisterInfo * TRI
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)
This header defines various interfaces for pass management in LLVM.
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file describes the interface of the MachineFunctionPass responsible for assigning the generic vi...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI Machine Scheduler interface.
static FunctionPass * useDefaultRegisterAllocator()
-regalloc=... command line option.
Target-Independent Code Generator Pass Configuration Options pass.
static std::unique_ptr< TargetLoweringObjectFile > createTLOF()
A manager for alias analyses.
void registerFunctionAnalysis()
Register a specific AA result.
void addAAResult(AAResultT &AAResult)
Register a specific AA result.
Legacy wrapper pass to provide the AMDGPUAAResult object.
Analysis pass providing a never-invalidated alias analysis result.
uint32_t getLDSSize() const
AMDGPUTargetMachine & getAMDGPUTargetMachine() const
std::unique_ptr< CSEConfigBase > getCSEConfig() const override
Returns the CSEConfig object to use for the current optimization level.
ScheduleDAGInstrs * createMachineScheduler(MachineSchedContext *C) const override
Create an instance of ScheduleDAGInstrs to be run within the standard MachineScheduler pass for this ...
bool isPassEnabled(const cl::opt< bool > &Opt, CodeGenOptLevel Level=CodeGenOptLevel::Default) const
Check if a pass is enabled given Opt option.
bool addPreISel() override
Methods with trivial inline returns are convenient points in the common codegen pass pipeline where t...
bool addInstSelector() override
addInstSelector - This method should install an instruction selector pass, which converts from LLVM c...
bool addGCPasses() override
addGCPasses - Add late codegen passes that analyze code for garbage collection.
void addStraightLineScalarOptimizationPasses()
AMDGPUPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
void addIRPasses() override
Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...
void addEarlyCSEOrGVNPass()
void addCodeGenPrepare() override
Add pass to prepare the LLVM IR for code generation.
static int64_t getNullPointerValue(unsigned AddrSpace)
Get the integer value of a null pointer in the given address space.
unsigned getAddressSpaceForPseudoSourceKind(unsigned Kind) const override
getAddressSpaceForPseudoSourceKind - Given the kind of memory (e.g.
const TargetSubtargetInfo * getSubtargetImpl() const
void registerDefaultAliasAnalyses(AAManager &) override
Allow the target to register alias analyses with the AAManager for use with the new pass manager.
~AMDGPUTargetMachine() override
std::pair< const Value *, unsigned > getPredicatedAddrSpace(const Value *V) const override
If the specified predicate checks whether a generic pointer falls within a specified address space,...
StringRef getFeatureString(const Function &F) const
static bool EnableLateStructurizeCFG
static bool EnableFunctionCalls
AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOptLevel OL)
bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
void registerPassBuilderCallbacks(PassBuilder &PB, bool PopulateClassToPassNames) override
Allow the target to modify the pass pipeline.
static bool EnableLowerModuleLDS
StringRef getGPUName(const Function &F) const
unsigned getAssumedAddrSpace(const Value *V) const override
If the specified generic pointer could be assumed as a pointer to a specific address space,...
static bool DisableStructurizer
StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
Allocate memory in an ever growing pool, as if by bump-pointer.
void removeDeadConstantUsers() const
If there are any dead constant users dangling off of this constant, remove them.
This pass is required by interprocedural register allocation.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
FunctionPass class - This class is used to implement most global optimizations.
@ SCHEDULE_LEGACYMAXOCCUPANCY
const SIRegisterInfo * getRegisterInfo() const override
TargetTransformInfo getTargetTransformInfo(const Function &F) const override
Get a TargetTransformInfo implementation for the target.
void registerMachineRegisterInfoCallback(MachineFunction &MF) const override
bool parseMachineFunctionInfo(const yaml::MachineFunctionInfo &, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) const override
Parse out the target's MachineFunctionInfo from the YAML reprsentation.
yaml::MachineFunctionInfo * convertFuncInfoToYAML(const MachineFunction &MF) const override
Allocate and initialize an instance of the YAML representation of the MachineFunctionInfo.
yaml::MachineFunctionInfo * createDefaultFuncInfoYAML() const override
Allocate and return a default initialized instance of the YAML representation for the MachineFunction...
TargetPassConfig * createPassConfig(PassManagerBase &PM) override
Create a pass configuration object to be used by addPassToEmitX methods for generating a pipeline of ...
GCNTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOptLevel OL, bool JIT)
MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override
Create the target's instance of MachineFunctionInfo.
Pass to remove unused function declarations.
This pass is responsible for selecting generic machine instructions to target-specific instructions.
A pass that internalizes all functions and variables other than those that must be preserved accordin...
This class describes a target machine that is implemented with the LLVM target-independent code gener...
This pass implements the localization mechanism described at the top of this file.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void addDelegate(Delegate *delegate)
MachineSchedRegistry provides a selection of available machine instruction schedulers.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
static const OptimizationLevel O0
Disable as many optimizations as possible.
unsigned getSpeedupLevel() const
static const OptimizationLevel O1
Optimize quickly without destroying debuggability.
This class provides access to building LLVM's passes.
void registerPipelineEarlySimplificationEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerPipelineStartEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerPeepholeEPCallback(const std::function< void(FunctionPassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerCGSCCOptimizerLateEPCallback(const std::function< void(CGSCCPassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerFullLinkTimeOptimizationLastEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
LLVM_ATTRIBUTE_MINSIZE void addPass(PassT &&Pass)
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
@ ExternalSymbolCallEntry
MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override
Create the target's instance of MachineFunctionInfo.
RegisterPassParser class - Handle the addition of new machine passes.
RegisterRegAllocBase class - Track the registration of register allocators.
FunctionPass *(*)() FunctionPassCtor
Wrapper class representing virtual and physical registers.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void reserveWWMRegister(Register Reg)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Represents a location in source code.
Represents a range in source code.
A ScheduleDAG for scheduling lists of MachineInstr.
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
const TargetInstrInfo * TII
Target instruction information.
const TargetRegisterInfo * TRI
Target processor register info.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
unsigned getMainFileID() const
const MemoryBuffer * getMemoryBuffer(unsigned i) const
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Triple TargetTriple
Triple string, CPU name, and target feature strings the TargetMachine instance is created with.
const Triple & getTargetTriple() const
const MCSubtargetInfo * getMCSubtargetInfo() const
StringRef getTargetFeatureString() const
StringRef getTargetCPU() const
std::unique_ptr< const MCSubtargetInfo > STI
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
std::unique_ptr< const MCRegisterInfo > MRI
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Target-Independent Code Generator Pass Configuration Options.
virtual void addCodeGenPrepare()
Add pass to prepare the LLVM IR for code generation.
virtual bool addILPOpts()
Add passes that optimize instruction level parallelism for out-of-order targets.
virtual void addPostRegAlloc()
This method may be implemented by targets that want to run passes after register allocation pass pipe...
CodeGenOptLevel getOptLevel() const
virtual void addOptimizedRegAlloc()
addOptimizedRegAlloc - Add passes related to register allocation.
virtual void addIRPasses()
Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...
virtual void addFastRegAlloc()
addFastRegAlloc - Add the minimum set of target-independent passes that are required for fast registe...
virtual void addMachineSSAOptimization()
addMachineSSAOptimization - Add standard passes that optimize machine instructions in SSA form.
void disablePass(AnalysisID PassID)
Allow the target to disable a specific standard pass by default.
AnalysisID addPass(AnalysisID PassID)
Utilities for targets to add passes to the pass manager.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ArchType getArch() const
Get the parsed architecture type of this triple.
LLVM Value Representation.
PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...
Interfaces for registering analysis passes, producing common pass manager configurations,...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
bool isFlatGlobalAddrSpace(unsigned AS)
bool isEntryFunctionCC(CallingConv::ID CC)
@ C
The default llvm calling convention, compatible with C.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
bool match(Val *V, const Pattern &P)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
This is an optimization pass for GlobalISel generic memory operations.
FunctionPass * createFlattenCFGPass()
void initializeSIFormMemoryClausesPass(PassRegistry &)
char & SIPreAllocateWWMRegsID
FunctionPass * createFastRegisterAllocator()
FastRegisterAllocation Pass - This pass register allocates as fast as possible.
char & EarlyMachineLICMID
This pass performs loop invariant code motion on machine instructions.
ImmutablePass * createAMDGPUAAWrapperPass()
char & PostRAHazardRecognizerID
PostRAHazardRecognizer - This pass runs the post-ra hazard recognizer.
FunctionPass * createAMDGPUSetWavePriorityPass()
void initializeAMDGPUInsertSingleUseVDSTPass(PassRegistry &)
void initializeGCNCreateVOPDPass(PassRegistry &)
ModulePass * createAMDGPUOpenCLEnqueuedBlockLoweringPass()
char & GCNPreRAOptimizationsID
char & GCLoweringID
GCLowering Pass - Used by gc.root to perform its default lowering operations.
void initializeGCNPreRAOptimizationsPass(PassRegistry &)
Pass * createLoadStoreVectorizerPass()
Create a legacy pass manager instance of the LoadStoreVectorizer pass.
void initializeGCNRewritePartialRegUsesPass(llvm::PassRegistry &)
void initializeAMDGPUAttributorLegacyPass(PassRegistry &)
void initializeAMDGPUDAGToDAGISelPass(PassRegistry &)
FunctionPass * createSIModeRegisterPass()
FunctionPass * createGreedyRegisterAllocator()
Greedy register allocation pass - This pass implements a global register allocator for optimized buil...
void initializeAMDGPUAAWrapperPassPass(PassRegistry &)
ModulePass * createAMDGPULowerBufferFatPointersPass()
void initializeR600ClauseMergePassPass(PassRegistry &)
void initializeSIModeRegisterPass(PassRegistry &)
ModulePass * createAMDGPUCtorDtorLoweringLegacyPass()
void initializeSIOptimizeVGPRLiveRangePass(PassRegistry &)
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
void initializeAMDGPULateCodeGenPreparePass(PassRegistry &)
void initializeAMDGPURewriteUndefForPHILegacyPass(PassRegistry &)
FunctionPass * createAMDGPUPreLegalizeCombiner(bool IsOptNone)
char & GCNRewritePartialRegUsesID
FunctionPass * createAMDGPUPostLegalizeCombiner(bool IsOptNone)
void initializeAMDGPUAnnotateUniformValuesPass(PassRegistry &)
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
void initializeSIShrinkInstructionsPass(PassRegistry &)
void initializeGCNPreRALongBranchRegPass(PassRegistry &)
char & SILoadStoreOptimizerID
std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation(AMDGPU::SchedulingPhase Phase)
Phase specifes whether or not this is a reentry into the IGroupLPDAGMutation.
FunctionPass * createNaryReassociatePass()
char & PatchableFunctionID
This pass implements the "patchable-function" attribute.
char & PostRASchedulerID
PostRAScheduler - This pass performs post register allocation scheduling.
void initializeR600ExpandSpecialInstrsPassPass(PassRegistry &)
void initializeR600PacketizerPass(PassRegistry &)
std::unique_ptr< ScheduleDAGMutation > createVOPDPairingMutation()
ModulePass * createAMDGPUAlwaysInlinePass(bool GlobalOpt=true)
void initializeSIPreEmitPeepholePass(PassRegistry &)
char & SILowerWWMCopiesID
void initializeSIFixVGPRCopiesPass(PassRegistry &)
void initializeAMDGPUGlobalISelDivergenceLoweringPass(PassRegistry &)
std::unique_ptr< CSEConfigBase > getStandardCSEConfigForOpt(CodeGenOptLevel Level)
Target & getTheR600Target()
The target for R600 GPUs.
char & MachineSchedulerID
MachineScheduler - This pass schedules machine instructions.
Pass * createStructurizeCFGPass(bool SkipUniformRegions=false)
When SkipUniformRegions is true the structizer will not structurize regions that only contain uniform...
void initializeAMDGPURemoveIncompatibleFunctionsPass(PassRegistry &)
void initializeSILowerWWMCopiesPass(PassRegistry &)
void initializeGCNNSAReassignPass(PassRegistry &)
char & PostMachineSchedulerID
PostMachineScheduler - This pass schedules machine instructions postRA.
void initializeSIInsertWaitcntsPass(PassRegistry &)
char & AMDGPUInsertSingleUseVDSTID
ScheduleDAGMILive * createGenericSchedLive(MachineSchedContext *C)
Create the standard converging machine scheduler.
char & SIFormMemoryClausesID
void initializeAMDGPULowerModuleLDSLegacyPass(PassRegistry &)
void initializeAMDGPUCtorDtorLoweringLegacyPass(PassRegistry &)
void initializeAMDGPURegBankCombinerPass(PassRegistry &)
void initializeSILoadStoreOptimizerPass(PassRegistry &)
void initializeSILateBranchLoweringPass(PassRegistry &)
void initializeSIPeepholeSDWAPass(PassRegistry &)
char & AMDGPUUnifyDivergentExitNodesID
FunctionPass * createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy)
char & ShadowStackGCLoweringID
ShadowStackGCLowering - Implements the custom lowering mechanism used by the shadow stack GC.
void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &)
void initializeAMDGPUExternalAAWrapperPass(PassRegistry &)
void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &)
char & AMDGPUPerfHintAnalysisID
char & SILowerSGPRSpillsID
CodeModel::Model getEffectiveCodeModel(std::optional< CodeModel::Model > CM, CodeModel::Model Default)
Helper method for getting the code model, returning Default if CM does not have a value.
char & SILateBranchLoweringPassID
char & BranchRelaxationPassID
BranchRelaxation - This pass replaces branches that need to jump further than is supported by a branc...
FunctionPass * createSinkingPass()
CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false, bool NoRerun=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
FunctionPass * createSIShrinkInstructionsPass()
void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &)
void initializeSIPostRABundlerPass(PassRegistry &)
void initializeAMDGPUPromoteAllocaToVectorPass(PassRegistry &)
Pass * createAMDGPUAttributorLegacyPass()
void initializeSIWholeQuadModePass(PassRegistry &)
std::unique_ptr< ScheduleDAGMutation > createStoreClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ReorderWhileClustering=false)
If ReorderWhileClustering is set to true, no attempt will be made to reduce reordering due to store c...
FunctionPass * createLoopDataPrefetchPass()
FunctionPass * createAMDGPULowerKernelArgumentsPass()
char & AMDGPUInsertDelayAluID
Pass * createAMDGPUAnnotateKernelFeaturesPass()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
std::unique_ptr< ScheduleDAGMutation > createAMDGPUMacroFusionDAGMutation()
Note that you have to add: DAG.addMutation(createAMDGPUMacroFusionDAGMutation()); to AMDGPUPassConfig...
char & StackMapLivenessID
StackMapLiveness - This pass analyses the register live-out set of stackmap/patchpoint intrinsics and...
char & SIOptimizeVGPRLiveRangeID
FunctionPass * createUnifyLoopExitsPass()
char & SIOptimizeExecMaskingPreRAID
FunctionPass * createFixIrreduciblePass()
char & FuncletLayoutID
This pass lays out funclets contiguously.
void initializeSIInsertHardClausesPass(PassRegistry &)
char & DetectDeadLanesID
This pass adds dead/undef flags after analyzing subregister lanes.
void initializeAMDGPUPostLegalizerCombinerPass(PassRegistry &)
CodeGenOptLevel
Code generation optimization level.
void initializeSIAnnotateControlFlowPass(PassRegistry &)
ModulePass * createAMDGPUPrintfRuntimeBinding()
void initializeSIMemoryLegalizerPass(PassRegistry &)
Pass * createAlwaysInlinerLegacyPass(bool InsertLifetime=true)
Create a legacy pass manager instance of a pass to inline and remove functions marked as "always_inli...
void initializeR600ControlFlowFinalizerPass(PassRegistry &)
void initializeAMDGPUImageIntrinsicOptimizerPass(PassRegistry &)
FunctionPass * createAMDGPUAnnotateUniformValues()
ModulePass * createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM=nullptr)
void initializeAMDGPUPreLegalizerCombinerPass(PassRegistry &)
FunctionPass * createAMDGPUPromoteAlloca()
FunctionPass * createSeparateConstOffsetFromGEPPass(bool LowerGEP=false)
char & EarlyIfConverterID
EarlyIfConverter - This pass performs if-conversion on SSA form by inserting cmov instructions.
char & SIPreEmitPeepholeID
ModulePass * createAMDGPURemoveIncompatibleFunctionsPass(const TargetMachine *)
FunctionPass * createSILowerI1CopiesPass()
void initializeGCNRegPressurePrinterPass(PassRegistry &)
void initializeAMDGPUArgumentUsageInfoPass(PassRegistry &)
FunctionPass * createBasicRegisterAllocator()
BasicRegisterAllocation Pass - This pass implements a degenerate global register allocator using the ...
void initializeGlobalISel(PassRegistry &)
Initialize all passes linked into the GlobalISel library.
void initializeSIPreAllocateWWMRegsPass(PassRegistry &)
ModulePass * createR600OpenCLImageTypeLoweringPass()
FunctionPass * createAMDGPUCodeGenPreparePass()
FunctionPass * createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a AMDGPU-specific.
Target & getTheGCNTarget()
The target for GCN GPUs.
void initializeAMDGPUAtomicOptimizerPass(PassRegistry &)
char & MachineCSEID
MachineCSE - This pass performs global CSE on machine instructions.
FunctionPass * createAMDGPURegBankCombiner(bool IsOptNone)
std::unique_ptr< ScheduleDAGMutation > createLoadClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ReorderWhileClustering=false)
If ReorderWhileClustering is set to true, no attempt will be made to reduce reordering due to store c...
void initializeSIOptimizeExecMaskingPreRAPass(PassRegistry &)
void initializeAMDGPUMarkLastScratchLoadPass(PassRegistry &)
char & LiveVariablesID
LiveVariables pass - This pass computes the set of blocks in which each variable is life and sets mac...
void initializeAMDGPUCodeGenPreparePass(PassRegistry &)
FunctionPass * createGVNPass(bool NoMemDepAnalysis=false)
Create a legacy GVN pass.
FunctionPass * createAMDGPURewriteUndefForPHILegacyPass()
void call_once(once_flag &flag, Function &&F, Args &&... ArgList)
Execute the function specified as a parameter once.
void initializeSILowerSGPRSpillsPass(PassRegistry &)
void initializeAMDGPULowerKernelAttributesPass(PassRegistry &)
char & SIInsertHardClausesID
FunctionPass * createAMDGPUMachineCFGStructurizerPass()
void initializeAMDGPUResourceUsageAnalysisPass(PassRegistry &)
void initializeSIFixSGPRCopiesPass(PassRegistry &)
FunctionPass * createInferAddressSpacesPass(unsigned AddressSpace=~0u)
char & VirtRegRewriterID
VirtRegRewriter pass.
void initializeSILowerI1CopiesPass(PassRegistry &)
char & SILowerControlFlowID
FunctionPass * createLowerSwitchPass()
FunctionPass * createVirtRegRewriter(bool ClearVirtRegs=true)
void initializeR600VectorRegMergerPass(PassRegistry &)
ImmutablePass * createExternalAAWrapperPass(std::function< void(Pass &, Function &, AAResults &)> Callback)
A wrapper pass around a callback which can be used to populate the AAResults in the AAResultsWrapperP...
void initializeSIOptimizeExecMaskingPass(PassRegistry &)
FunctionPass * createAMDGPUGlobalISelDivergenceLoweringPass()
FunctionPass * createSIMemoryLegalizerPass()
void initializeSIFoldOperandsPass(PassRegistry &)
void initializeSILowerControlFlowPass(PassRegistry &)
char & TwoAddressInstructionPassID
TwoAddressInstruction - This pass reduces two-address instructions to use two operands.
void initializeAMDGPURegBankSelectPass(PassRegistry &)
FunctionPass * createAtomicExpandLegacyPass()
AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...
MCRegisterInfo * createGCNMCRegisterInfo(AMDGPUDwarfFlavour DwarfFlavour)
FunctionPass * createStraightLineStrengthReducePass()
FunctionPass * createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *)
void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry &)
void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &)
FunctionPass * createSIInsertWaitcntsPass()
FunctionPass * createEarlyCSEPass(bool UseMemorySSA=false)
void initializeGCNDPPCombinePass(PassRegistry &)
char & PHIEliminationID
PHIElimination - This pass eliminates machine instruction PHI nodes by inserting copy instructions.
bool parseNamedRegisterReference(PerFunctionMIParsingState &PFS, Register &Reg, StringRef Src, SMDiagnostic &Error)
FunctionPass * createAMDGPULateCodeGenPreparePass()
char & AMDGPUMarkLastScratchLoadID
char & RenameIndependentSubregsID
This pass detects subregister lanes in a virtual register that are used independently of other lanes ...
std::unique_ptr< ScheduleDAGMutation > createAMDGPUExportClusteringDAGMutation()
void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry &)
void initializeAMDGPUPromoteAllocaPass(PassRegistry &)
void initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(PassRegistry &)
void initializeAMDGPUInsertDelayAluPass(PassRegistry &)
char & SIOptimizeExecMaskingID
void initializeAMDGPUUnifyMetadataPass(PassRegistry &)
FunctionPass * createSIAnnotateControlFlowPass()
Create the annotation pass.
void initializeAMDGPUAlwaysInlinePass(PassRegistry &)
char & DeadMachineInstructionElimID
DeadMachineInstructionElim - This pass removes dead machine instructions.
char & GCNPreRALongBranchRegID
void initializeAMDGPUPromoteKernelArgumentsPass(PassRegistry &)
ArgDescriptor PrivateSegmentBuffer
ArgDescriptor WorkGroupIDY
ArgDescriptor WorkGroupIDZ
ArgDescriptor PrivateSegmentSize
ArgDescriptor ImplicitArgPtr
ArgDescriptor PrivateSegmentWaveByteOffset
ArgDescriptor WorkGroupInfo
ArgDescriptor WorkItemIDZ
ArgDescriptor WorkItemIDY
ArgDescriptor LDSKernelId
ArgDescriptor KernargSegmentPtr
ArgDescriptor WorkItemIDX
ArgDescriptor FlatScratchInit
ArgDescriptor DispatchPtr
ArgDescriptor ImplicitBufferPtr
ArgDescriptor WorkGroupIDX
static ArgDescriptor createStack(unsigned Offset, unsigned Mask=~0u)
static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ IEEE
IEEE-754 denormal numbers preserved.
DenormalModeKind Output
Denormal flushing mode for floating point instruction results in the default floating point environme...
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
RegisterTargetMachine - Helper template for registering a target machine implementation,...
bool DX10Clamp
Used by the vector ALU to force DX10-style treatment of NaNs: when set, clamp NaN to zero; otherwise,...
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
bool IEEE
Floating point opcodes that support exception flag gathering quiet and propagate signaling NaN inputs...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
The llvm::once_flag structure.
Targets should override this in a way that mirrors the implementation of llvm::MachineFunctionInfo.
StringValue SGPRForEXECCopy
SmallVector< StringValue > WWMReservedRegs
StringValue FrameOffsetReg
StringValue LongBranchReservedReg
StringValue VGPRForAGPRCopy
std::optional< SIArgumentInfo > ArgInfo
StringValue ScratchRSrcReg
StringValue StackPtrOffsetReg
bool FP64FP16OutputDenormals
bool FP64FP16InputDenormals
A wrapper around std::string which contains a source range that's being set during parsing.