90#define DEBUG_TYPE "load-store-vectorizer"
92STATISTIC(NumVectorInstructions,
"Number of vector accesses generated");
93STATISTIC(NumScalarsVectorized,
"Number of scalar accesses vectorized");
107using ChainID =
const Value *;
124 :
F(
F), AA(AA), AC(AC), DT(DT), SE(SE),
TTI(
TTI),
130 unsigned getPointerAddressSpace(
Value *
I);
132 static const unsigned MaxDepth = 3;
136 unsigned Depth = 0)
const;
137 bool lookThroughComplexAddresses(
Value *PtrA,
Value *PtrB,
APInt PtrDelta,
138 unsigned Depth)
const;
139 bool lookThroughSelects(
Value *PtrA,
Value *PtrB,
const APInt &PtrDelta,
140 unsigned Depth)
const;
147 std::pair<BasicBlock::iterator, BasicBlock::iterator>
169 std::pair<InstrListMap, InstrListMap> collectInstructions(
BasicBlock *BB);
173 bool vectorizeChains(InstrListMap &Map);
192 bool accessIsMisaligned(
unsigned SzInBytes,
unsigned AddressSpace,
193 Align Alignment,
unsigned &RelativeSpeed);
196class LoadStoreVectorizerLegacyPass :
public FunctionPass {
207 return "GPU Load and Store Vectorizer";
222char LoadStoreVectorizerLegacyPass::ID = 0;
225 "Vectorize load and Store instructions",
false,
false)
236 return new LoadStoreVectorizerLegacyPass();
239bool LoadStoreVectorizerLegacyPass::runOnFunction(
Function &
F) {
241 if (skipFunction(
F) ||
F.hasFnAttribute(Attribute::NoImplicitFloat))
244 AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
245 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
246 ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
248 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
F);
251 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
F);
253 Vectorizer
V(
F, AA, AC, DT, SE,
TTI);
259 if (
F.hasFnAttribute(Attribute::NoImplicitFloat))
268 Vectorizer V(
F, AA, AC, DT, SE,
TTI);
269 bool Changed = V.run();
283bool Vectorizer::run() {
284 bool Changed =
false;
288 InstrListMap LoadRefs, StoreRefs;
289 std::tie(LoadRefs, StoreRefs) = collectInstructions(BB);
290 Changed |= vectorizeChains(LoadRefs);
291 Changed |= vectorizeChains(StoreRefs);
297unsigned Vectorizer::getPointerAddressSpace(
Value *
I) {
299 return L->getPointerAddressSpace();
301 return S->getPointerAddressSpace();
306bool Vectorizer::isConsecutiveAccess(
Value *
A,
Value *
B) {
309 unsigned ASA = getPointerAddressSpace(
A);
310 unsigned ASB = getPointerAddressSpace(
B);
313 if (!PtrA || !PtrB || (ASA != ASB))
321 DL.getTypeStoreSize(PtrATy) !=
DL.getTypeStoreSize(PtrBTy) ||
326 unsigned PtrBitWidth =
DL.getPointerSizeInBits(ASA);
327 APInt Size(PtrBitWidth,
DL.getTypeStoreSize(PtrATy));
329 return areConsecutivePointers(PtrA, PtrB,
Size);
332bool Vectorizer::areConsecutivePointers(
Value *PtrA,
Value *PtrB,
334 unsigned PtrBitWidth =
DL.getPointerTypeSizeInBits(PtrA->
getType());
335 APInt OffsetA(PtrBitWidth, 0);
336 APInt OffsetB(PtrBitWidth, 0);
340 unsigned NewPtrBitWidth =
DL.getTypeStoreSizeInBits(PtrA->
getType());
342 if (NewPtrBitWidth !=
DL.getTypeStoreSizeInBits(PtrB->
getType()))
349 assert(OffsetA.getSignificantBits() <= NewPtrBitWidth &&
350 OffsetB.getSignificantBits() <= NewPtrBitWidth);
352 OffsetA = OffsetA.sextOrTrunc(NewPtrBitWidth);
353 OffsetB = OffsetB.sextOrTrunc(NewPtrBitWidth);
356 APInt OffsetDelta = OffsetB - OffsetA;
361 return OffsetDelta == PtrDelta;
365 APInt BaseDelta = PtrDelta - OffsetDelta;
386 return lookThroughComplexAddresses(PtrA, PtrB, BaseDelta,
Depth);
397 unsigned MatchingOpIdxB,
bool Signed) {
415 AddOpB->
getOpcode() == Instruction::Add &&
419 Value *OtherOperandA = AddOpA->
getOperand(MatchingOpIdxA == 1 ? 0 : 1);
420 Value *OtherOperandB = AddOpB->
getOperand(MatchingOpIdxB == 1 ? 0 : 1);
421 Instruction *OtherInstrA = dyn_cast<Instruction>(OtherOperandA);
422 Instruction *OtherInstrB = dyn_cast<Instruction>(OtherOperandB);
424 if (OtherInstrB && OtherInstrB->
getOpcode() == Instruction::Add &&
426 isa<ConstantInt>(OtherInstrB->
getOperand(1))) {
428 cast<ConstantInt>(OtherInstrB->
getOperand(1))->getSExtValue();
429 if (OtherInstrB->
getOperand(0) == OtherOperandA &&
434 if (OtherInstrA && OtherInstrA->
getOpcode() == Instruction::Add &&
436 isa<ConstantInt>(OtherInstrA->
getOperand(1))) {
438 cast<ConstantInt>(OtherInstrA->
getOperand(1))->getSExtValue();
439 if (OtherInstrA->
getOperand(0) == OtherOperandB &&
445 if (OtherInstrA && OtherInstrB &&
446 OtherInstrA->
getOpcode() == Instruction::Add &&
447 OtherInstrB->
getOpcode() == Instruction::Add &&
450 isa<ConstantInt>(OtherInstrA->
getOperand(1)) &&
451 isa<ConstantInt>(OtherInstrB->
getOperand(1))) {
453 cast<ConstantInt>(OtherInstrA->
getOperand(1))->getSExtValue();
455 cast<ConstantInt>(OtherInstrB->
getOperand(1))->getSExtValue();
464bool Vectorizer::lookThroughComplexAddresses(
Value *PtrA,
Value *PtrB,
466 unsigned Depth)
const {
467 auto *GEPA = dyn_cast<GetElementPtrInst>(PtrA);
468 auto *GEPB = dyn_cast<GetElementPtrInst>(PtrB);
470 return lookThroughSelects(PtrA, PtrB, PtrDelta,
Depth);
474 if (GEPA->getNumOperands() != GEPB->getNumOperands() ||
475 GEPA->getPointerOperand() != GEPB->getPointerOperand())
479 for (
unsigned I = 0,
E = GEPA->getNumIndices() - 1;
I <
E; ++
I) {
499 if (PtrDelta.
urem(Stride) != 0)
505 if (!isa<SExtInst>(OpA) && !isa<ZExtInst>(OpA))
508 bool Signed = isa<SExtInst>(OpA);
512 OpB = dyn_cast<Instruction>(OpB->
getOperand(0));
521 if (OpB->
getOpcode() == Instruction::Add &&
523 IdxDiff.
sle(cast<ConstantInt>(OpB->
getOperand(1))->getSExtValue()) &&
529 OpA = dyn_cast<Instruction>(ValA);
530 if (!Safe && OpA && OpA->
getOpcode() == Instruction::Add &&
537 for (
unsigned MatchingOpIdxA : {0, 1})
538 for (
unsigned MatchingOpIdxB : {0, 1})
556 if (BitsAllowedToBeSet.
ult(IdxDiff))
564 return X == OffsetSCEVB;
567bool Vectorizer::lookThroughSelects(
Value *PtrA,
Value *PtrB,
568 const APInt &PtrDelta,
569 unsigned Depth)
const {
573 if (
auto *SelectA = dyn_cast<SelectInst>(PtrA)) {
574 if (
auto *SelectB = dyn_cast<SelectInst>(PtrB)) {
575 return SelectA->getCondition() == SelectB->getCondition() &&
576 areConsecutivePointers(SelectA->getTrueValue(),
577 SelectB->getTrueValue(), PtrDelta,
Depth) &&
578 areConsecutivePointers(SelectA->getFalseValue(),
579 SelectB->getFalseValue(), PtrDelta,
Depth);
590 while (!Worklist.
empty()) {
593 for (
int i = 0; i < NumOperands; i++) {
595 if (!IM || IM->
getOpcode() == Instruction::PHI)
604 InstructionsToMove.
insert(IM);
611 for (
auto BBI =
I->getIterator(),
E =
I->getParent()->end(); BBI !=
E;
613 if (!InstructionsToMove.
count(&*BBI))
622std::pair<BasicBlock::iterator, BasicBlock::iterator>
629 unsigned NumFound = 0;
636 FirstInstr =
I.getIterator();
638 if (NumFound == Chain.
size()) {
639 LastInstr =
I.getIterator();
645 return std::make_pair(FirstInstr, ++LastInstr);
652 assert(PtrOperand &&
"Instruction must have a pointer operand.");
661 I->eraseFromParent();
666 unsigned ElementSizeBits) {
667 unsigned ElementSizeBytes = ElementSizeBits / 8;
668 unsigned SizeBytes = ElementSizeBytes * Chain.
size();
669 unsigned NumLeft = (SizeBytes - (SizeBytes % 4)) / ElementSizeBytes;
670 if (NumLeft == Chain.
size()) {
671 if ((NumLeft & 1) == 0)
675 }
else if (NumLeft == 0)
677 return std::make_pair(Chain.
slice(0, NumLeft), Chain.
slice(NumLeft));
686 bool IsLoadChain = isa<LoadInst>(Chain[0]);
691 "All elements of Chain must be loads, or all must be stores.");
694 "All elements of Chain must be loads, or all must be stores.");
699 if ((isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
is_contained(Chain, &
I)) {
704 LLVM_DEBUG(
dbgs() <<
"LSV: Found instruction may not transfer execution: "
708 if (
I.mayReadOrWriteMemory())
713 unsigned ChainInstrIdx = 0;
716 for (
unsigned E = ChainInstrs.
size(); ChainInstrIdx <
E; ++ChainInstrIdx) {
717 Instruction *ChainInstr = ChainInstrs[ChainInstrIdx];
721 if (BarrierMemoryInstr && BarrierMemoryInstr->
comesBefore(ChainInstr))
728 if (BarrierMemoryInstr && BarrierMemoryInstr->
comesBefore(MemInstr))
731 auto *MemLoad = dyn_cast<LoadInst>(MemInstr);
732 auto *ChainLoad = dyn_cast<LoadInst>(ChainInstr);
733 if (MemLoad && ChainLoad)
738 auto IsInvariantLoad = [](
const LoadInst *LI) ->
bool {
739 return LI->hasMetadata(LLVMContext::MD_invariant_load);
748 (ChainLoad && IsInvariantLoad(ChainLoad)))
752 if (MemInstr->comesBefore(ChainInstr) ||
753 (MemLoad && IsInvariantLoad(MemLoad)))
761 dbgs() <<
"LSV: Found alias:\n"
762 " Aliasing instruction:\n"
763 <<
" " << *MemInstr <<
'\n'
764 <<
" Aliased instruction and pointer:\n"
765 <<
" " << *ChainInstr <<
'\n'
770 BarrierMemoryInstr = MemInstr;
778 if (IsLoadChain && BarrierMemoryInstr) {
790 ChainInstrs.
begin(), ChainInstrs.
begin() + ChainInstrIdx);
791 unsigned ChainIdx = 0;
792 for (
unsigned ChainLen = Chain.size(); ChainIdx < ChainLen; ++ChainIdx) {
793 if (!VectorizableChainInstrs.count(Chain[ChainIdx]))
796 return Chain.slice(0, ChainIdx);
801 if (
const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
808 return Sel->getCondition();
813std::pair<InstrListMap, InstrListMap>
814Vectorizer::collectInstructions(
BasicBlock *BB) {
815 InstrListMap LoadRefs;
816 InstrListMap StoreRefs;
819 if (!
I.mayReadOrWriteMemory())
822 if (
LoadInst *LI = dyn_cast<LoadInst>(&
I)) {
830 Type *Ty = LI->getType();
836 unsigned TySize =
DL.getTypeSizeInBits(Ty);
837 if ((TySize % 8) != 0)
847 Value *
Ptr = LI->getPointerOperand();
848 unsigned AS =
Ptr->getType()->getPointerAddressSpace();
851 unsigned VF = VecRegSize / TySize;
855 if (TySize > VecRegSize / 2 ||
861 LoadRefs[
ID].push_back(LI);
862 }
else if (
StoreInst *SI = dyn_cast<StoreInst>(&
I)) {
870 Type *Ty =
SI->getValueOperand()->getType();
883 unsigned TySize =
DL.getTypeSizeInBits(Ty);
884 if ((TySize % 8) != 0)
888 unsigned AS =
Ptr->getType()->getPointerAddressSpace();
891 unsigned VF = VecRegSize / TySize;
895 if (TySize > VecRegSize / 2 ||
901 StoreRefs[
ID].push_back(SI);
905 return {LoadRefs, StoreRefs};
908bool Vectorizer::vectorizeChains(InstrListMap &Map) {
909 bool Changed =
false;
911 for (
const std::pair<ChainID, InstrList> &Chain : Map) {
912 unsigned Size = Chain.second.size();
919 for (
unsigned CI = 0, CE =
Size; CI <
CE; CI += 64) {
920 unsigned Len = std::min<unsigned>(CE - CI, 64);
922 Changed |= vectorizeInstructions(Chunk);
931 <<
" instructions.\n");
933 int ConsecutiveChain[64];
937 for (
int i = 0, e = Instrs.
size(); i < e; ++i) {
938 ConsecutiveChain[i] = -1;
939 for (
int j = e - 1;
j >= 0; --
j) {
944 if (ConsecutiveChain[i] != -1) {
945 int CurDistance = std::abs(ConsecutiveChain[i] - i);
946 int NewDistance = std::abs(ConsecutiveChain[i] - j);
947 if (j < i || NewDistance > CurDistance)
953 ConsecutiveChain[i] =
j;
958 bool Changed =
false;
961 for (
int Head : Heads) {
962 if (InstructionsProcessed.
count(Instrs[Head]))
964 bool LongerChainExists =
false;
965 for (
unsigned TIt = 0; TIt < Tails.
size(); TIt++)
966 if (Head == Tails[TIt] &&
967 !InstructionsProcessed.
count(Instrs[Heads[TIt]])) {
968 LongerChainExists =
true;
971 if (LongerChainExists)
979 if (InstructionsProcessed.
count(Instrs[
I]))
983 I = ConsecutiveChain[
I];
986 bool Vectorized =
false;
987 if (isa<LoadInst>(*
Operands.begin()))
988 Vectorized = vectorizeLoadChain(
Operands, &InstructionsProcessed);
990 Vectorized = vectorizeStoreChain(
Operands, &InstructionsProcessed);
992 Changed |= Vectorized;
998bool Vectorizer::vectorizeStoreChain(
1001 StoreInst *S0 = cast<StoreInst>(Chain[0]);
1004 Type *StoreTy =
nullptr;
1006 StoreTy = cast<StoreInst>(
I)->getValueOperand()->getType();
1012 DL.getTypeSizeInBits(StoreTy));
1016 assert(StoreTy &&
"Failed to find store type");
1018 unsigned Sz =
DL.getTypeSizeInBits(StoreTy);
1021 unsigned VF = VecRegSize / Sz;
1022 unsigned ChainSize = Chain.size();
1026 InstructionsProcessed->
insert(Chain.begin(), Chain.end());
1031 if (NewChain.
empty()) {
1033 InstructionsProcessed->
insert(Chain.begin(), Chain.end());
1036 if (NewChain.
size() == 1) {
1044 ChainSize = Chain.size();
1048 unsigned EltSzInBytes = Sz / 8;
1049 unsigned SzInBytes = EltSzInBytes * ChainSize;
1052 auto *VecStoreTy = dyn_cast<FixedVectorType>(StoreTy);
1055 Chain.size() * VecStoreTy->getNumElements());
1062 if (ChainSize > VF || (VF != TargetVF && TargetVF < ChainSize)) {
1063 LLVM_DEBUG(
dbgs() <<
"LSV: Chain doesn't match with the vector factor."
1064 " Creating two separate arrays.\n");
1065 bool Vectorized =
false;
1067 vectorizeStoreChain(Chain.slice(0, TargetVF), InstructionsProcessed);
1069 vectorizeStoreChain(Chain.slice(TargetVF), InstructionsProcessed);
1074 dbgs() <<
"LSV: Stores to vectorize:\n";
1076 dbgs() <<
" " << *
I <<
"\n";
1081 InstructionsProcessed->
insert(Chain.begin(), Chain.end());
1084 unsigned RelativeSpeed;
1085 if (accessIsMisaligned(SzInBytes, AS, Alignment, RelativeSpeed)) {
1087 unsigned SpeedBefore;
1088 accessIsMisaligned(EltSzInBytes, AS, Alignment, SpeedBefore);
1089 if (SpeedBefore > RelativeSpeed)
1092 auto Chains = splitOddVectorElts(Chain, Sz);
1093 bool Vectorized =
false;
1094 Vectorized |= vectorizeStoreChain(Chains.first, InstructionsProcessed);
1095 Vectorized |= vectorizeStoreChain(Chains.second, InstructionsProcessed);
1101 DL, S0,
nullptr, &DT);
1102 if (NewAlign >= Alignment)
1103 Alignment = NewAlign;
1109 auto Chains = splitOddVectorElts(Chain, Sz);
1110 bool Vectorized =
false;
1111 Vectorized |= vectorizeStoreChain(Chains.first, InstructionsProcessed);
1112 Vectorized |= vectorizeStoreChain(Chains.second, InstructionsProcessed);
1117 std::tie(First,
Last) = getBoundaryInstrs(Chain);
1123 unsigned VecWidth = VecStoreTy->getNumElements();
1124 for (
unsigned I = 0,
E = Chain.size();
I !=
E; ++
I) {
1126 for (
unsigned J = 0, NE = VecStoreTy->getNumElements(); J != NE; ++J) {
1127 unsigned NewIdx = J +
I * VecWidth;
1134 Builder.CreateInsertElement(Vec, Extract,
Builder.getInt32(NewIdx));
1139 for (
unsigned I = 0,
E = Chain.size();
I !=
E; ++
I) {
1158 eraseInstructions(Chain);
1159 ++NumVectorInstructions;
1160 NumScalarsVectorized += Chain.size();
1164bool Vectorizer::vectorizeLoadChain(
1167 LoadInst *L0 = cast<LoadInst>(Chain[0]);
1170 Type *LoadTy =
nullptr;
1171 for (
const auto &V : Chain) {
1172 LoadTy = cast<LoadInst>(V)->getType();
1178 DL.getTypeSizeInBits(LoadTy));
1182 assert(LoadTy &&
"Can't determine LoadInst type from chain");
1184 unsigned Sz =
DL.getTypeSizeInBits(LoadTy);
1187 unsigned VF = VecRegSize / Sz;
1188 unsigned ChainSize = Chain.size();
1192 InstructionsProcessed->
insert(Chain.begin(), Chain.end());
1197 if (NewChain.
empty()) {
1199 InstructionsProcessed->
insert(Chain.begin(), Chain.end());
1202 if (NewChain.
size() == 1) {
1210 ChainSize = Chain.size();
1214 unsigned EltSzInBytes = Sz / 8;
1215 unsigned SzInBytes = EltSzInBytes * ChainSize;
1217 auto *VecLoadTy = dyn_cast<FixedVectorType>(LoadTy);
1220 Chain.size() * VecLoadTy->getNumElements());
1227 if (ChainSize > VF || (VF != TargetVF && TargetVF < ChainSize)) {
1228 LLVM_DEBUG(
dbgs() <<
"LSV: Chain doesn't match with the vector factor."
1229 " Creating two separate arrays.\n");
1230 bool Vectorized =
false;
1232 vectorizeLoadChain(Chain.slice(0, TargetVF), InstructionsProcessed);
1234 vectorizeLoadChain(Chain.slice(TargetVF), InstructionsProcessed);
1240 InstructionsProcessed->
insert(Chain.begin(), Chain.end());
1243 unsigned RelativeSpeed;
1244 if (accessIsMisaligned(SzInBytes, AS, Alignment, RelativeSpeed)) {
1246 unsigned SpeedBefore;
1247 accessIsMisaligned(EltSzInBytes, AS, Alignment, SpeedBefore);
1248 if (SpeedBefore > RelativeSpeed)
1251 auto Chains = splitOddVectorElts(Chain, Sz);
1252 bool Vectorized =
false;
1253 Vectorized |= vectorizeLoadChain(Chains.first, InstructionsProcessed);
1254 Vectorized |= vectorizeLoadChain(Chains.second, InstructionsProcessed);
1260 DL, L0,
nullptr, &DT);
1261 if (NewAlign >= Alignment)
1262 Alignment = NewAlign;
1268 auto Chains = splitOddVectorElts(Chain, Sz);
1269 bool Vectorized =
false;
1270 Vectorized |= vectorizeLoadChain(Chains.first, InstructionsProcessed);
1271 Vectorized |= vectorizeLoadChain(Chains.second, InstructionsProcessed);
1276 dbgs() <<
"LSV: Loads to vectorize:\n";
1285 std::tie(First,
Last) = getBoundaryInstrs(Chain);
1286 Builder.SetInsertPoint(&*First);
1294 for (
unsigned I = 0,
E = Chain.size();
I !=
E; ++
I) {
1299 unsigned VecWidth = VecLoadTy->getNumElements();
1301 llvm::to_vector<8>(llvm::seq<int>(
I * VecWidth, (
I + 1) * VecWidth));
1307 if (
V->getType() != CV->
getType()) {
1323 Instruction *BCInst = dyn_cast<Instruction>(Bitcast);
1326 eraseInstructions(Chain);
1328 ++NumVectorInstructions;
1329 NumScalarsVectorized += Chain.size();
1333bool Vectorizer::accessIsMisaligned(
unsigned SzInBytes,
unsigned AddressSpace,
1334 Align Alignment,
unsigned &RelativeSpeed) {
1336 if (Alignment.
value() % SzInBytes == 0)
1341 Alignment, &RelativeSpeed);
1342 LLVM_DEBUG(
dbgs() <<
"LSV: Target said misaligned is allowed? " << Allows
1343 <<
" with relative speed = " << RelativeSpeed <<
'\n';);
1344 return !Allows || !RelativeSpeed;
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static const unsigned MaxDepth
static bool checkNoWrapFlags(Instruction *I, bool Signed)
static ChainID getChainID(const Value *Ptr)
static const unsigned StackAdjustedAlignment
static bool checkIfSafeAddSequence(const APInt &IdxDiff, Instruction *AddOpA, unsigned MatchingOpIdxA, Instruction *AddOpB, unsigned MatchingOpIdxB, bool Signed)
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
This file provides utility analysis objects describing memory locations.
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)
Check whether or not an instruction may read or write the optionally specified memory location.
Class for arbitrary precision integers.
APInt udiv(const APInt &RHS) const
Unsigned division operation.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
APInt zext(unsigned width) const
Zero extend to a new width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
APInt trunc(unsigned width) const
Truncate to new width.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
void negate()
Negate this APInt in place.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
int64_t getSExtValue() const
Get sign extended value.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
InstListType::iterator iterator
Instruction iterators...
Represents analyses that only rely on functions' control flow.
A parsed version of the target data layout string in and methods for querying it.
Analysis pass which computes a DominatorTree.
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Legacy wrapper pass to provide the GlobalsAAResult object.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const BasicBlock * getParent() const
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
This class implements a map that also provides access to all stored values in a deterministic order.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
Legacy wrapper pass to provide the SCEVAAResult object.
This class represents an analyzed expression in the program.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVMContext & getContext() const
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
Value * getOperand() const
Type * getIndexedType() const
self_iterator getIterator()
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ CE
Windows NT (Windows on ARM)
This is an optimization pass for GlobalISel generic memory operations.
Pass * createLoadStoreVectorizerPass()
Create a legacy pass manager instance of the LoadStoreVectorizer pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
iterator_range< po_iterator< T > > post_order(const T &G)
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Align getOrEnforceKnownAlignment(Value *V, MaybeAlign PrefAlign, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to ensure that the alignment of V is at least PrefAlign bytes.
bool isModSet(const ModRefInfo MRI)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isModOrRefSet(const ModRefInfo MRI)
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, OptimizationRemarkEmitter *ORE=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, ScalarEvolution &SE, bool CheckType=true)
Returns true if the memory operations A and B are consecutive.
constexpr unsigned BitWidth
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
gep_type_iterator gep_type_begin(const User *GEP)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry &)
Type * getLoadStoreType(Value *I)
A helper function that returns the type of a load or store instruction.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.