LLVM  14.0.0git
MachineFunctionSplitter.cpp
Go to the documentation of this file.
1 //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // \file
10 // Uses profile information to split out cold blocks.
11 //
12 // This pass splits out cold machine basic blocks from the parent function. This
13 // implementation leverages the basic block section framework. Blocks marked
14 // cold by this pass are grouped together in a separate section prefixed with
15 // ".text.unlikely.*". The linker can then group these together as a cold
16 // section. The split part of the function is a contiguous region identified by
17 // the symbol "foo.cold". Grouping all cold blocks across functions together
18 // decreases fragmentation and improves icache and itlb utilization. Note that
19 // the overall changes to the binary size are negligible; only a small number of
20 // additional jump instructions may be introduced.
21 //
22 // For the original RFC of this pass please see
23 // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ
24 //===----------------------------------------------------------------------===//
25 
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/Statistic.h"
35 #include "llvm/CodeGen/Passes.h"
36 #include "llvm/IR/Function.h"
37 #include "llvm/IR/Module.h"
38 #include "llvm/InitializePasses.h"
40 
41 using namespace llvm;
42 
43 // FIXME: This cutoff value is CPU dependent and should be moved to
44 // TargetTransformInfo once we consider enabling this on other platforms.
45 // The value is expressed as a ProfileSummaryInfo integer percentile cutoff.
46 // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split.
47 // The default was empirically determined to be optimal when considering cutoff
48 // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on
49 // Intel CPUs.
50 static cl::opt<unsigned>
51  PercentileCutoff("mfs-psi-cutoff",
52  cl::desc("Percentile profile summary cutoff used to "
53  "determine cold blocks. Unused if set to zero."),
54  cl::init(999950), cl::Hidden);
55 
57  "mfs-count-threshold",
58  cl::desc(
59  "Minimum number of times a block must be executed to be retained."),
60  cl::init(1), cl::Hidden);
61 
62 namespace {
63 
64 class MachineFunctionSplitter : public MachineFunctionPass {
65 public:
66  static char ID;
67  MachineFunctionSplitter() : MachineFunctionPass(ID) {
69  }
70 
71  StringRef getPassName() const override {
72  return "Machine Function Splitter Transformation";
73  }
74 
75  void getAnalysisUsage(AnalysisUsage &AU) const override;
76 
77  bool runOnMachineFunction(MachineFunction &F) override;
78 };
79 } // end anonymous namespace
80 
81 static bool isColdBlock(const MachineBasicBlock &MBB,
82  const MachineBlockFrequencyInfo *MBFI,
83  ProfileSummaryInfo *PSI) {
85  if (!Count.hasValue())
86  return true;
87 
88  if (PercentileCutoff > 0) {
89  return PSI->isColdCountNthPercentile(PercentileCutoff, *Count);
90  }
91  return (*Count < ColdCountThreshold);
92 }
93 
94 bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) {
95  // TODO: We only target functions with profile data. Static information may
96  // also be considered but we don't see performance improvements yet.
97  if (!MF.getFunction().hasProfileData())
98  return false;
99 
100  // TODO: We don't split functions where a section attribute has been set
101  // since the split part may not be placed in a contiguous region. It may also
102  // be more beneficial to augment the linker to ensure contiguous layout of
103  // split functions within the same section as specified by the attribute.
104  if (MF.getFunction().hasSection() ||
105  MF.getFunction().hasFnAttribute("implicit-section-name"))
106  return false;
107 
108  // We don't want to proceed further for cold functions
109  // or functions of unknown hotness. Lukewarm functions have no prefix.
110  Optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
111  if (SectionPrefix.hasValue() &&
112  (SectionPrefix.getValue().equals("unlikely") ||
113  SectionPrefix.getValue().equals("unknown"))) {
114  return false;
115  }
116 
117  // Renumbering blocks here preserves the order of the blocks as
118  // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort
119  // blocks. Preserving the order of blocks is essential to retaining decisions
120  // made by prior passes such as MachineBlockPlacement.
121  MF.RenumberBlocks();
123  auto *MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
124  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
125 
127  for (auto &MBB : MF) {
128  if (MBB.isEntryBlock())
129  continue;
130 
131  if (MBB.isEHPad())
132  LandingPads.push_back(&MBB);
133  else if (isColdBlock(MBB, MBFI, PSI))
135  }
136 
137  // We only split out eh pads if all of them are cold.
138  bool HasHotLandingPads = false;
139  for (const MachineBasicBlock *LP : LandingPads) {
140  if (!isColdBlock(*LP, MBFI, PSI))
141  HasHotLandingPads = true;
142  }
143  if (!HasHotLandingPads) {
144  for (MachineBasicBlock *LP : LandingPads)
145  LP->setSectionID(MBBSectionID::ColdSectionID);
146  }
147 
148  auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) {
149  return X.getSectionID().Type < Y.getSectionID().Type;
150  };
152 
153  return true;
154 }
155 
156 void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const {
160 }
161 
163 INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",
164  "Split machine functions using profile information", false,
165  false)
166 
168  return new MachineFunctionSplitter();
169 }
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::sortBasicBlocksAndUpdateBranches
void sortBasicBlocksAndUpdateBranches(MachineFunction &MF, MachineBasicBlockComparator MBBCmp)
Definition: BasicBlockSections.cpp:296
llvm::createMachineFunctionSplitterPass
MachineFunctionPass * createMachineFunctionSplitterPass()
createMachineFunctionSplitterPass - This pass splits machine functions using profile information.
llvm::MachineBlockFrequencyInfo::getBlockProfileCount
Optional< uint64_t > getBlockProfileCount(const MachineBasicBlock *MBB) const
Definition: MachineBlockFrequencyInfo.cpp:234
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Statistic.h
llvm::MachineBasicBlock::isEntryBlock
bool isEntryBlock() const
Returns true if this is the entry block of the function.
Definition: MachineBasicBlock.cpp:290
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
MachineBasicBlock.h
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
Module.h
llvm::Optional< uint64_t >
INITIALIZE_PASS
INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter", "Split machine functions using profile information", false, false) MachineFunctionPass *llvm
Definition: MachineFunctionSplitter.cpp:163
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::Optional::hasValue
constexpr bool hasValue() const
Definition: Optional.h:288
llvm::MachineBasicBlock::setSectionID
void setSectionID(MBBSectionID V)
Sets the section ID for this basic block.
Definition: MachineBasicBlock.h:583
CommandLine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::MachineBlockFrequencyInfo
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
Definition: MachineBlockFrequencyInfo.h:33
llvm::initializeMachineFunctionSplitterPass
void initializeMachineFunctionSplitterPass(PassRegistry &)
Y
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
ColdCountThreshold
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
llvm::Function::getSectionPrefix
Optional< StringRef > getSectionPrefix() const
Get the section prefix for this function.
Definition: Function.cpp:1950
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::StringRef::equals
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:187
llvm::ProfileSummaryInfo
Analysis providing profile information.
Definition: ProfileSummaryInfo.h:39
Passes.h
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:626
llvm::cl::opt
Definition: CommandLine.h:1434
llvm::GlobalObject::hasSection
bool hasSection() const
Check if this global has a custom object file section.
Definition: GlobalObject.h:104
PercentileCutoff
static cl::opt< unsigned > PercentileCutoff("mfs-psi-cutoff", cl::desc("Percentile profile summary cutoff used to " "determine cold blocks. Unused if set to zero."), cl::init(999950), cl::Hidden)
ProfileSummaryInfo.h
llvm::MachineModuleInfoWrapperPass
Definition: MachineModuleInfo.h:279
llvm::BasicBlockSection::Preset
@ Preset
BasicBlockSectionUtils.h
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::ProfileSummaryInfoWrapperPass
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:185
MachineFunctionPass.h
isColdBlock
static bool isColdBlock(const MachineBasicBlock &MBB, const MachineBlockFrequencyInfo *MBFI, ProfileSummaryInfo *PSI)
Definition: MachineFunctionSplitter.cpp:81
MachineModuleInfo.h
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::MachineBasicBlock::isEHPad
bool isEHPad() const
Returns true if the block is a landing pad.
Definition: MachineBasicBlock.h:526
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:592
llvm::MBBSectionID::ColdSectionID
const static MBBSectionID ColdSectionID
Definition: MachineBasicBlock.h:64
Function.h
llvm::Function::hasProfileData
bool hasProfileData(bool IncludeSynthetic=false) const
Return true if the function is annotated with profile data.
Definition: Function.h:302
SmallVector.h
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::cl::desc
Definition: CommandLine.h:414
MachineFunction.h
llvm::MachineFunction::setBBSectionsType
void setBBSectionsType(BasicBlockSection V)
Definition: MachineFunction.h:615
InitializePasses.h
MachineBlockFrequencyInfo.h
llvm::MachineFunction::RenumberBlocks
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.
Definition: MachineFunction.cpp:294
llvm::ProfileSummaryInfo::isColdCountNthPercentile
bool isColdCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered cold with regard to a given cold percentile cutoff value.
Definition: ProfileSummaryInfo.cpp:313
llvm::Optional::getValue
constexpr const T & getValue() const LLVM_LVALUE_FUNCTION
Definition: Optional.h:282
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37