28#include "llvm/IR/IntrinsicsNVPTX.h"
49 cl::desc(
"Disable load/store vectorizer"),
54 "disable-nvptx-require-structured-cfg",
55 cl::desc(
"Transitional flag to turn off NVPTX's requirement on preserving "
56 "structured CFG. The requirement should be disabled only when "
57 "unexpected regressions happen."),
63 "Use 32-bit pointers for accessing const/local/shared address spaces."),
110 std::string Ret =
"e";
114 else if (UseShortPointers)
115 Ret +=
"-p3:32:32-p4:32:32-p5:32:32";
117 Ret +=
"-i64:64-i128:128-v16:16-v32:32-n16:32:64";
125 std::optional<Reloc::Model> RM,
126 std::optional<CodeModel::Model> CM,
148void NVPTXTargetMachine32::anchor() {}
153 std::optional<Reloc::Model> RM,
154 std::optional<CodeModel::Model> CM,
158void NVPTXTargetMachine64::anchor() {}
163 std::optional<Reloc::Model> RM,
164 std::optional<CodeModel::Model> CM,
176 return getTM<NVPTXTargetMachine>();
179 void addIRPasses()
override;
180 bool addInstSelector()
override;
181 void addPreRegAlloc()
override;
182 void addPostRegAlloc()
override;
183 void addMachineSSAOptimization()
override;
185 FunctionPass *createTargetRegisterAllocator(
bool)
override;
186 void addFastRegAlloc()
override;
187 void addOptimizedRegAlloc()
override;
189 bool addRegAssignAndRewriteFast()
override {
193 bool addRegAssignAndRewriteOptimized()
override {
200 void addEarlyCSEOrGVNPass();
203 void addAddressSpaceInferencePasses();
206 void addStraightLineScalarOptimizationPasses();
212 return new NVPTXPassConfig(*
this, PM);
218 return NVPTXMachineFunctionInfo::create<NVPTXMachineFunctionInfo>(
Allocator,
234 if (
PassName ==
"nvvm-intr-range") {
246 if (AAName ==
"nvptx-aa") {
256 if (
PassName ==
"nvptx-lower-ctor-dtor") {
260 if (
PassName ==
"generic-to-nvvm") {
283std::pair<const Value *, unsigned>
285 if (
auto *II = dyn_cast<IntrinsicInst>(V)) {
286 switch (II->getIntrinsicID()) {
287 case Intrinsic::nvvm_isspacep_const:
289 case Intrinsic::nvvm_isspacep_global:
291 case Intrinsic::nvvm_isspacep_local:
293 case Intrinsic::nvvm_isspacep_shared:
294 case Intrinsic::nvvm_isspacep_shared_cluster:
300 return std::make_pair(
nullptr, -1);
303void NVPTXPassConfig::addEarlyCSEOrGVNPass() {
310void NVPTXPassConfig::addAddressSpaceInferencePasses() {
319void NVPTXPassConfig::addStraightLineScalarOptimizationPasses() {
328 addEarlyCSEOrGVNPass();
336void NVPTXPassConfig::addIRPasses() {
376 addAddressSpaceInferencePasses();
377 addStraightLineScalarOptimizationPasses();
398 addEarlyCSEOrGVNPass();
405bool NVPTXPassConfig::addInstSelector() {
412 if (!
ST.hasImageHandles())
418void NVPTXPassConfig::addPreRegAlloc() {
423void NVPTXPassConfig::addPostRegAlloc() {
433FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(
bool) {
437void NVPTXPassConfig::addFastRegAlloc() {
442void NVPTXPassConfig::addOptimizedRegAlloc() {
453 printAndVerify(
"After Machine Scheduling");
461 printAndVerify(
"After StackSlotColoring");
464void NVPTXPassConfig::addMachineSSAOptimization() {
467 printAndVerify(
"After Pre-RegAlloc TailDuplicate");
486 printAndVerify(
"After codegen DCE pass");
492 printAndVerify(
"After ILP optimizations");
498 printAndVerify(
"After Machine LICM, CSE and Sinking passes");
501 printAndVerify(
"After codegen peephole optimization pass");
#define LLVM_EXTERNAL_VISIBILITY
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
static std::string computeDataLayout()
This is the NVPTX address space based alias analysis pass.
static cl::opt< bool > DisableLoadStoreVectorizer("disable-nvptx-load-store-vectorizer", cl::desc("Disable load/store vectorizer"), cl::init(false), cl::Hidden)
static cl::opt< bool > DisableRequireStructuredCFG("disable-nvptx-require-structured-cfg", cl::desc("Transitional flag to turn off NVPTX's requirement on preserving " "structured CFG. The requirement should be disabled only when " "unexpected regressions happen."), cl::init(false), cl::Hidden)
static cl::opt< bool > UseShortPointersOpt("nvptx-short-ptr", cl::desc("Use 32-bit pointers for accessing const/local/shared address spaces."), cl::init(false), cl::Hidden)
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget()
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)
Target-Independent Code Generator Pass Configuration Options pass.
static bool is64Bit(const char *name)
static const char PassName[]
A manager for alias analyses.
void registerFunctionAnalysis()
Register a specific AA result.
void addAAResult(AAResultT &AAResult)
Register a specific AA result.
A container for analyses that lazily runs them and caches their results.
bool registerPass(PassBuilderT &&PassBuilder)
Register an analysis pass with the manager.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Allocate memory in an ever growing pool, as if by bump-pointer.
FunctionPass class - This class is used to implement most global optimizations.
This class describes a target machine that is implemented with the LLVM target-independent code gener...
Legacy wrapper pass to provide the NVPTXAAResult object.
Analysis pass providing a never-invalidated alias analysis result.
Lower llvm.global_ctors and llvm.global_dtors to special kernels.
unsigned int getSmVersion() const
NVPTXTargetMachine32(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOpt::Level OL, bool JIT)
NVPTXTargetMachine64(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOpt::Level OL, bool JIT)
TargetTransformInfo getTargetTransformInfo(const Function &F) const override
Get a TargetTransformInfo implementation for the target.
void registerDefaultAliasAnalyses(AAManager &AAM) override
Allow the target to register alias analyses with the AAManager for use with the new pass manager.
std::pair< const Value *, unsigned > getPredicatedAddrSpace(const Value *V) const override
If the specified predicate checks whether a generic pointer falls within a specified address space,...
void registerPassBuilderCallbacks(PassBuilder &PB) override
Allow the target to modify the pass pipeline.
~NVPTXTargetMachine() override
NVPTXTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOpt::Level OP, bool is64bit)
MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override
Create the target's instance of MachineFunctionInfo.
TargetPassConfig * createPassConfig(PassManagerBase &PM) override
Create a pass configuration object to be used by addPassToEmitX methods for generating a pipeline of ...
This class provides access to building LLVM's passes.
void registerPipelineStartEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerParseAACallback(const std::function< bool(StringRef Name, AAManager &AA)> &C)
Register a callback for parsing an AliasAnalysis Name to populate the given AAManager AA.
void registerAnalysisRegistrationCallback(const std::function< void(CGSCCAnalysisManager &)> &C)
{{@ Register callbacks for analysis registration with this PassBuilder instance.
void registerPipelineParsingCallback(const std::function< bool(StringRef Name, CGSCCPassManager &, ArrayRef< PipelineElement >)> &C)
{{@ Register pipeline parsing callbacks with this pass builder instance.
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT &&Pass)
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
StringRef - Represent a constant reference to a string, i.e.
void setRequiresStructuredCFG(bool Value)
std::unique_ptr< const MCSubtargetInfo > STI
Target-Independent Code Generator Pass Configuration Options.
virtual void addIRPasses()
Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...
TargetSubtargetInfo - Generic base class for all target subtargets.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
LLVM Value Representation.
PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...
Interfaces for registering analysis passes, producing common pass manager configurations,...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Level
Code generation optimization level.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void initializeNVPTXLowerAllocaPass(PassRegistry &)
char & EarlyMachineLICMID
This pass performs loop invariant code motion on machine instructions.
ModulePass * createNVPTXAssignValidGlobalNamesPass()
MachineFunctionPass * createNVPTXReplaceImageHandlesPass()
void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry &)
Pass * createLoadStoreVectorizerPass()
Create a legacy pass manager instance of the LoadStoreVectorizer pass.
char & RegisterCoalescerID
RegisterCoalescer - This pass merges live ranges to eliminate copies.
char & OptimizePHIsID
OptimizePHIs - This pass optimizes machine instruction PHIs to take advantage of opportunities create...
char & EarlyTailDuplicateID
Duplicate blocks with unconditional branches into tails of their predecessors.
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
ModulePass * createGenericToNVVMLegacyPass()
FunctionPass * createNVVMReflectPass(unsigned int SmVersion)
void initializeNVPTXLowerAggrCopiesPass(PassRegistry &)
void initializeNVPTXExternalAAWrapperPass(PassRegistry &)
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
FunctionPass * createAtomicExpandPass()
AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...
MachineFunctionPass * createNVPTXPrologEpilogPass()
MachineFunctionPass * createNVPTXProxyRegErasurePass()
void initializeNVPTXDAGToDAGISelPass(PassRegistry &)
char & TailDuplicateID
TailDuplicate - Duplicate blocks with unconditional branches into tails of their predecessors.
FunctionPass * createNaryReassociatePass()
char & PatchableFunctionID
This pass implements the "patchable-function" attribute.
char & PostRASchedulerID
PostRAScheduler - This pass performs post register allocation scheduling.
MachineFunctionPass * createNVPTXPeephole()
void initializeNVVMReflectPass(PassRegistry &)
char & MachineSchedulerID
MachineScheduler - This pass schedules machine instructions.
char & PeepholeOptimizerID
PeepholeOptimizer - This pass performs peephole optimizations - like extension and comparison elimina...
char & LiveDebugValuesID
LiveDebugValues pass.
char & PrologEpilogCodeInserterID
PrologEpilogCodeInserter - This pass inserts prolog and epilog code, and eliminates abstract frame re...
void initializeGenericToNVVMLegacyPassPass(PassRegistry &)
void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &)
void initializeNVPTXLowerArgsPass(PassRegistry &)
char & MachineLoopInfoID
MachineLoopInfo - This pass is a loop analysis pass.
FunctionPass * createNVPTXLowerArgsPass()
CodeModel::Model getEffectiveCodeModel(std::optional< CodeModel::Model > CM, CodeModel::Model Default)
Helper method for getting the code model, returning Default if CM does not have a value.
char & ShrinkWrapID
ShrinkWrap pass. Look for the best place to insert save and restore.
char & MachineLateInstrsCleanupID
MachineLateInstrsCleanup - This pass removes redundant identical instructions after register allocati...
void initializeNVPTXAAWrapperPassPass(PassRegistry &)
FunctionPass * createNVPTXImageOptimizerPass()
FunctionPass * createNVPTXLowerAllocaPass()
FunctionPass * createSpeculativeExecutionPass()
char & StackMapLivenessID
StackMapLiveness - This pass analyses the register live-out set of stackmap/patchpoint intrinsics and...
char & FuncletLayoutID
This pass lays out funclets contiguously.
FunctionPass * createAllocaHoisting()
void initializeNVVMIntrRangePass(PassRegistry &)
char & StackColoringID
StackSlotColoring - This pass performs stack coloring and merging.
char & PostRAMachineSinkingID
This pass perform post-ra machine sink for COPY instructions.
char & StackSlotColoringID
StackSlotColoring - This pass performs stack slot coloring.
FunctionPass * createSeparateConstOffsetFromGEPPass(bool LowerGEP=false)
FunctionPass * createLowerAggrCopies()
char & ProcessImplicitDefsID
ProcessImpicitDefs pass - This pass removes IMPLICIT_DEFs.
char & MachineCSEID
MachineCSE - This pass performs global CSE on machine instructions.
FunctionPass * createNVPTXAtomicLowerPass()
ModulePass * createNVPTXCtorDtorLoweringLegacyPass()
char & LiveVariablesID
LiveVariables pass - This pass computes the set of blocks in which each variable is life and sets mac...
FunctionPass * createGVNPass(bool NoMemDepAnalysis=false)
Create a legacy GVN pass.
void initializeNVPTXAllocaHoistingPass(PassRegistry &)
Target & getTheNVPTXTarget64()
FunctionPass * createInferAddressSpacesPass(unsigned AddressSpace=~0u)
void initializeNVPTXProxyRegErasurePass(PassRegistry &)
ImmutablePass * createNVPTXAAWrapperPass()
ImmutablePass * createExternalAAWrapperPass(std::function< void(Pass &, Function &, AAResults &)> Callback)
A wrapper pass around a callback which can be used to populate the AAResults in the AAResultsWrapperP...
char & TwoAddressInstructionPassID
TwoAddressInstruction - This pass reduces two-address instructions to use two operands.
char & LocalStackSlotAllocationID
LocalStackSlotAllocation - This pass assigns local frame indices to stack slots relative to one anoth...
FunctionPass * createStraightLineStrengthReducePass()
FunctionPass * createEarlyCSEPass(bool UseMemorySSA=false)
char & PHIEliminationID
PHIElimination - This pass eliminates machine instruction PHI nodes by inserting copy instructions.
FunctionPass * createSROAPass(bool PreserveCFG=true)
void initializeNVPTXAtomicLowerPass(PassRegistry &)
char & MachineCopyPropagationID
MachineCopyPropagation - This pass performs copy propagation on machine instructions.
char & DeadMachineInstructionElimID
DeadMachineInstructionElim - This pass removes dead machine instructions.
FunctionPass * createNVPTXISelDag(NVPTXTargetMachine &TM, llvm::CodeGenOpt::Level OptLevel)
createNVPTXISelDag - This pass converts a legalized DAG into a NVPTX-specific DAG,...
Target & getTheNVPTXTarget32()
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
RegisterTargetMachine - Helper template for registering a target machine implementation,...