35#include "llvm/IR/IntrinsicsARM.h"
48#define DEBUG_TYPE "arm-mve-gather-scatter-lowering"
52 cl::desc(
"Enable the generation of masked gathers and scatters"));
67 return "MVE gather/scatter lowering";
82 bool isLegalTypeAndAlignment(
unsigned NumElements,
unsigned ElemSize,
85 void lookThroughBitcast(
Value *&
Ptr);
98 int computeScale(
unsigned GEPElemSize,
unsigned MemoryElemSize);
101 std::optional<int64_t> getIfConst(
const Value *V);
105 std::pair<Value *, int64_t> getVarAndConst(
Value *Inst,
int TypeScale);
115 int64_t Increment = 0);
119 int64_t Increment = 0);
128 int64_t Increment = 0);
132 int64_t Increment = 0);
153 void pushOutAdd(
PHINode *&Phi,
Value *OffsSecondOperand,
unsigned StartIndex);
155 void pushOutMulShl(
unsigned Opc,
PHINode *&Phi,
Value *IncrementPerRound,
156 Value *OffsSecondOperand,
unsigned LoopIncrement,
162char MVEGatherScatterLowering::ID = 0;
165 "MVE gather/scattering lowering pass",
false,
false)
168 return new MVEGatherScatterLowering();
171bool MVEGatherScatterLowering::isLegalTypeAndAlignment(
unsigned NumElements,
174 if (((NumElements == 4 &&
175 (ElemSize == 32 || ElemSize == 16 || ElemSize == 8)) ||
176 (NumElements == 8 && (ElemSize == 16 || ElemSize == 8)) ||
177 (NumElements == 16 && ElemSize == 8)) &&
178 Alignment >= ElemSize / 8)
180 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: instruction does not have "
181 <<
"valid alignment or vector type \n");
196 unsigned TargetElemSize = 128 / TargetElemCount;
197 unsigned OffsetElemSize = cast<FixedVectorType>(Offsets->getType())
199 ->getScalarSizeInBits();
200 if (OffsetElemSize != TargetElemSize || OffsetElemSize != 32) {
201 Constant *ConstOff = dyn_cast<Constant>(Offsets);
204 int64_t TargetElemMaxSize = (1ULL << TargetElemSize);
205 auto CheckValueSize = [TargetElemMaxSize](
Value *OffsetElem) {
206 ConstantInt *OConst = dyn_cast<ConstantInt>(OffsetElem);
210 if (SExtValue >= TargetElemMaxSize || SExtValue < 0)
214 if (isa<FixedVectorType>(ConstOff->
getType())) {
215 for (
unsigned i = 0; i < TargetElemCount; i++) {
220 if (!CheckValueSize(ConstOff))
231 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
Ptr)) {
234 computeScale(
GEP->getSourceElementType()->getPrimitiveSizeInBits(),
236 return Scale == -1 ? nullptr :
V;
254Value *MVEGatherScatterLowering::decomposeGEP(
Value *&Offsets,
259 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: no getelementpointer "
263 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: getelementpointer found."
264 <<
" Looking at intrinsic for base + vector of offsets\n");
265 Value *GEPPtr =
GEP->getPointerOperand();
268 !isa<FixedVectorType>(
Offsets->getType()))
271 if (
GEP->getNumOperands() != 2) {
272 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: getelementptr with too many"
273 <<
" operands. Expanding.\n");
277 unsigned OffsetsElemCount =
278 cast<FixedVectorType>(
Offsets->getType())->getNumElements();
282 ZExtInst *ZextOffs = dyn_cast<ZExtInst>(Offsets);
289 if (!ZextOffs || cast<FixedVectorType>(ZextOffs->
getDestTy())
291 ->getScalarSizeInBits() != 32)
297 if (Ty !=
Offsets->getType()) {
302 Offsets =
Builder.CreateZExt(Offsets, VectorType::getInteger(Ty));
306 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: found correct offsets\n");
310void MVEGatherScatterLowering::lookThroughBitcast(
Value *&
Ptr) {
312 if (
auto *BitCast = dyn_cast<BitCastInst>(
Ptr)) {
313 auto *BCTy = cast<FixedVectorType>(BitCast->getType());
314 auto *BCSrcTy = cast<FixedVectorType>(BitCast->getOperand(0)->getType());
315 if (BCTy->getNumElements() == BCSrcTy->getNumElements()) {
318 Ptr = BitCast->getOperand(0);
323int MVEGatherScatterLowering::computeScale(
unsigned GEPElemSize,
324 unsigned MemoryElemSize) {
327 if (GEPElemSize == 32 && MemoryElemSize == 32)
329 else if (GEPElemSize == 16 && MemoryElemSize == 16)
331 else if (GEPElemSize == 8)
333 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: incorrect scale. Can't "
334 <<
"create intrinsic\n");
338std::optional<int64_t> MVEGatherScatterLowering::getIfConst(
const Value *V) {
339 const Constant *
C = dyn_cast<Constant>(V);
340 if (
C &&
C->getSplatValue())
341 return std::optional<int64_t>{
C->getUniqueInteger().getSExtValue()};
342 if (!isa<Instruction>(V))
343 return std::optional<int64_t>{};
346 if (
I->getOpcode() == Instruction::Add ||
I->getOpcode() == Instruction::Or ||
347 I->getOpcode() == Instruction::Mul ||
348 I->getOpcode() == Instruction::Shl) {
349 std::optional<int64_t> Op0 = getIfConst(
I->getOperand(0));
350 std::optional<int64_t> Op1 = getIfConst(
I->getOperand(1));
352 return std::optional<int64_t>{};
353 if (
I->getOpcode() == Instruction::Add)
354 return std::optional<int64_t>{*Op0 + *Op1};
355 if (
I->getOpcode() == Instruction::Mul)
356 return std::optional<int64_t>{*Op0 * *Op1};
357 if (
I->getOpcode() == Instruction::Shl)
358 return std::optional<int64_t>{*Op0 << *Op1};
359 if (
I->getOpcode() == Instruction::Or)
360 return std::optional<int64_t>{*Op0 | *Op1};
362 return std::optional<int64_t>{};
368 return I->getOpcode() == Instruction::Or &&
372std::pair<Value *, int64_t>
373MVEGatherScatterLowering::getVarAndConst(
Value *Inst,
int TypeScale) {
374 std::pair<Value *, int64_t> ReturnFalse =
375 std::pair<Value *, int64_t>(
nullptr, 0);
379 if (
Add ==
nullptr ||
384 std::optional<int64_t>
Const;
386 if ((Const = getIfConst(
Add->getOperand(0))))
387 Summand =
Add->getOperand(1);
388 else if ((Const = getIfConst(
Add->getOperand(1))))
389 Summand =
Add->getOperand(0);
394 int64_t Immediate = *
Const << TypeScale;
395 if (Immediate > 512 || Immediate < -512 || Immediate % 4 != 0)
398 return std::pair<Value *, int64_t>(Summand, Immediate);
402 using namespace PatternMatch;
403 LLVM_DEBUG(
dbgs() <<
"masked gathers: checking transform preconditions\n"
409 auto *Ty = cast<FixedVectorType>(
I->getType());
411 Align Alignment = cast<ConstantInt>(
I->getArgOperand(1))->getAlignValue();
413 Value *PassThru =
I->getArgOperand(3);
418 lookThroughBitcast(
Ptr);
419 assert(
Ptr->getType()->isVectorTy() &&
"Unexpected pointer type");
423 Builder.SetCurrentDebugLocation(
I->getDebugLoc());
429 Load = tryCreateMaskedGatherOffset(
I,
Ptr, Root, Builder);
431 Load = tryCreateMaskedGatherBase(
I,
Ptr, Builder);
435 if (!isa<UndefValue>(PassThru) && !
match(PassThru,
m_Zero())) {
436 LLVM_DEBUG(
dbgs() <<
"masked gathers: found non-trivial passthru - "
437 <<
"creating select\n");
447 I->eraseFromParent();
449 LLVM_DEBUG(
dbgs() <<
"masked gathers: successfully built masked gather\n"
454Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
456 using namespace PatternMatch;
457 auto *Ty = cast<FixedVectorType>(
I->getType());
458 LLVM_DEBUG(
dbgs() <<
"masked gathers: loading from vector of pointers\n");
464 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base,
465 {Ty,
Ptr->getType()},
468 return Builder.CreateIntrinsic(
469 Intrinsic::arm_mve_vldr_gather_base_predicated,
470 {Ty,
Ptr->getType(),
Mask->getType()},
474Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
476 using namespace PatternMatch;
477 auto *Ty = cast<FixedVectorType>(
I->getType());
478 LLVM_DEBUG(
dbgs() <<
"masked gathers: loading from vector of pointers with "
485 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base_wb,
486 {Ty,
Ptr->getType()},
489 return Builder.CreateIntrinsic(
490 Intrinsic::arm_mve_vldr_gather_base_wb_predicated,
491 {Ty,
Ptr->getType(),
Mask->getType()},
495Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
497 using namespace PatternMatch;
499 Type *MemoryTy =
I->getType();
500 Type *ResultTy = MemoryTy;
506 bool TruncResult =
false;
508 if (
I->hasOneUse()) {
513 if (isa<SExtInst>(
User) &&
520 }
else if (isa<ZExtInst>(
User) &&
523 << *ResultTy <<
"\n");
534 128 / cast<FixedVectorType>(ResultTy)->getNumElements());
536 LLVM_DEBUG(
dbgs() <<
"masked gathers: Small input type, truncing to: "
537 << *ResultTy <<
"\n");
542 LLVM_DEBUG(
dbgs() <<
"masked gathers: Extend needed but not provided "
543 "from the correct type. Expanding\n");
551 Ptr, Offsets, Scale, cast<FixedVectorType>(ResultTy), MemoryTy, Builder);
560 Intrinsic::arm_mve_vldr_gather_offset_predicated,
566 Intrinsic::arm_mve_vldr_gather_offset,
572 Load = TruncInst::Create(Instruction::Trunc, Load, MemoryTy);
579 using namespace PatternMatch;
580 LLVM_DEBUG(
dbgs() <<
"masked scatters: checking transform preconditions\n"
586 Value *Input =
I->getArgOperand(0);
588 Align Alignment = cast<ConstantInt>(
I->getArgOperand(2))->getAlignValue();
589 auto *Ty = cast<FixedVectorType>(Input->
getType());
595 lookThroughBitcast(
Ptr);
596 assert(
Ptr->getType()->isVectorTy() &&
"Unexpected pointer type");
600 Builder.SetCurrentDebugLocation(
I->getDebugLoc());
604 Store = tryCreateMaskedScatterOffset(
I,
Ptr, Builder);
606 Store = tryCreateMaskedScatterBase(
I,
Ptr, Builder);
610 LLVM_DEBUG(
dbgs() <<
"masked scatters: successfully built masked scatter\n"
612 I->eraseFromParent();
616Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
618 using namespace PatternMatch;
619 Value *Input =
I->getArgOperand(0);
620 auto *Ty = cast<FixedVectorType>(Input->
getType());
628 LLVM_DEBUG(
dbgs() <<
"masked scatters: storing to a vector of pointers\n");
630 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base,
634 return Builder.CreateIntrinsic(
635 Intrinsic::arm_mve_vstr_scatter_base_predicated,
637 {
Ptr,
Builder.getInt32(Increment), Input, Mask});
640Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
642 using namespace PatternMatch;
643 Value *Input =
I->getArgOperand(0);
644 auto *Ty = cast<FixedVectorType>(Input->
getType());
645 LLVM_DEBUG(
dbgs() <<
"masked scatters: storing to a vector of pointers "
646 <<
"with writeback\n");
652 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base_wb,
656 return Builder.CreateIntrinsic(
657 Intrinsic::arm_mve_vstr_scatter_base_wb_predicated,
659 {
Ptr,
Builder.getInt32(Increment), Input, Mask});
662Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
664 using namespace PatternMatch;
665 Value *Input =
I->getArgOperand(0);
668 Type *MemoryTy = InputTy;
670 LLVM_DEBUG(
dbgs() <<
"masked scatters: getelementpointer found. Storing"
671 <<
" to base + vector of offsets\n");
674 if (
TruncInst *Trunc = dyn_cast<TruncInst>(Input)) {
675 Value *PreTrunc = Trunc->getOperand(0);
679 InputTy = PreTruncTy;
682 bool ExtendInput =
false;
690 128 / cast<FixedVectorType>(InputTy)->getNumElements());
692 LLVM_DEBUG(
dbgs() <<
"masked scatters: Small input type, will extend:\n"
696 LLVM_DEBUG(
dbgs() <<
"masked scatters: cannot create scatters for "
697 "non-standard input types. Expanding.\n");
704 Ptr, Offsets, Scale, cast<FixedVectorType>(InputTy), MemoryTy, Builder);
709 Input =
Builder.CreateZExt(Input, InputTy);
711 return Builder.CreateIntrinsic(
712 Intrinsic::arm_mve_vstr_scatter_offset_predicated,
717 Builder.getInt32(Scale), Mask});
719 return Builder.CreateIntrinsic(
720 Intrinsic::arm_mve_vstr_scatter_offset,
727Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
730 if (
I->getIntrinsicID() == Intrinsic::masked_gather)
731 Ty = cast<FixedVectorType>(
I->getType());
733 Ty = cast<FixedVectorType>(
I->getArgOperand(0)->getType());
739 Loop *
L = LI->getLoopFor(
I->getParent());
750 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: trying to build incrementing "
751 "wb gather/scatter\n");
756 computeScale(
DL->getTypeSizeInBits(
GEP->getOperand(0)->getType()),
757 DL->getTypeSizeInBits(
GEP->getType()) /
758 cast<FixedVectorType>(
GEP->getType())->getNumElements());
762 if (
GEP->hasOneUse()) {
766 if (
auto *Load = tryCreateIncrementingWBGatScat(
I, BasePtr, Offsets,
771 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: trying to build incrementing "
772 "non-wb gather/scatter\n");
774 std::pair<Value *, int64_t>
Add = getVarAndConst(Offsets, TypeScale);
775 if (
Add.first ==
nullptr)
778 int64_t Immediate =
Add.second;
782 Instruction::Shl, OffsetsIncoming,
787 Instruction::Add, ScaledOffsets,
792 cast<VectorType>(ScaledOffsets->
getType())->getElementType())),
795 if (
I->getIntrinsicID() == Intrinsic::masked_gather)
796 return tryCreateMaskedGatherBase(
I, OffsetsIncoming, Builder, Immediate);
798 return tryCreateMaskedScatterBase(
I, OffsetsIncoming, Builder, Immediate);
801Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
806 Loop *
L = LI->getLoopFor(
I->getParent());
810 if (Phi ==
nullptr ||
Phi->getNumIncomingValues() != 2 ||
811 Phi->getParent() !=
L->getHeader() ||
Phi->getNumUses() != 2)
818 unsigned IncrementIndex =
819 Phi->getIncomingBlock(0) ==
L->getLoopLatch() ? 0 : 1;
821 Offsets =
Phi->getIncomingValue(IncrementIndex);
823 std::pair<Value *, int64_t>
Add = getVarAndConst(Offsets, TypeScale);
824 if (
Add.first ==
nullptr)
827 int64_t Immediate =
Add.second;
828 if (OffsetsIncoming != Phi)
833 Builder.SetInsertPoint(&
Phi->getIncomingBlock(1 - IncrementIndex)->back());
835 cast<FixedVectorType>(OffsetsIncoming->
getType())->getNumElements();
839 Instruction::Shl,
Phi->getIncomingValue(1 - IncrementIndex),
841 "ScaledIndex", &
Phi->getIncomingBlock(1 - IncrementIndex)->back());
844 Instruction::Add, ScaledOffsets,
849 cast<VectorType>(ScaledOffsets->
getType())->getElementType())),
850 "StartIndex", &
Phi->getIncomingBlock(1 - IncrementIndex)->back());
853 Instruction::Sub, OffsetsIncoming,
855 "PreIncrementStartIndex",
856 &
Phi->getIncomingBlock(1 - IncrementIndex)->back());
857 Phi->setIncomingValue(1 - IncrementIndex, OffsetsIncoming);
863 if (
I->getIntrinsicID() == Intrinsic::masked_gather) {
865 Value *
Load = tryCreateMaskedGatherBaseWB(
I, Phi, Builder, Immediate);
874 EndResult = NewInduction =
875 tryCreateMaskedScatterBaseWB(
I, Phi, Builder, Immediate);
880 Phi->setIncomingValue(IncrementIndex, NewInduction);
885void MVEGatherScatterLowering::pushOutAdd(
PHINode *&Phi,
886 Value *OffsSecondOperand,
887 unsigned StartIndex) {
888 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: optimising add instruction\n");
890 &cast<Instruction>(
Phi->getIncomingBlock(StartIndex)->back());
893 Instruction::Add,
Phi->getIncomingValue(StartIndex), OffsSecondOperand,
894 "PushedOutAdd", InsertionPoint);
895 unsigned IncrementIndex = StartIndex == 0 ? 1 : 0;
898 Phi->addIncoming(NewIndex,
Phi->getIncomingBlock(StartIndex));
899 Phi->addIncoming(
Phi->getIncomingValue(IncrementIndex),
900 Phi->getIncomingBlock(IncrementIndex));
901 Phi->removeIncomingValue(IncrementIndex);
902 Phi->removeIncomingValue(StartIndex);
905void MVEGatherScatterLowering::pushOutMulShl(
unsigned Opcode,
PHINode *&Phi,
906 Value *IncrementPerRound,
907 Value *OffsSecondOperand,
908 unsigned LoopIncrement,
910 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: optimising mul instruction\n");
915 Phi->getIncomingBlock(LoopIncrement == 1 ? 0 : 1)->back());
920 Phi->getIncomingValue(LoopIncrement == 1 ? 0 : 1),
921 OffsSecondOperand,
"PushedOutMul", InsertionPoint);
925 OffsSecondOperand,
"Product", InsertionPoint);
928 Instruction::Add, Phi, Product,
"IncrementPushedOutMul",
929 cast<Instruction>(
Phi->getIncomingBlock(LoopIncrement)->back())
932 Phi->addIncoming(StartIndex,
933 Phi->getIncomingBlock(LoopIncrement == 1 ? 0 : 1));
934 Phi->addIncoming(NewIncrement,
Phi->getIncomingBlock(LoopIncrement));
935 Phi->removeIncomingValue((
unsigned)0);
936 Phi->removeIncomingValue((
unsigned)0);
942 if (
I->hasNUses(0)) {
946 for (
User *U :
I->users()) {
947 if (!isa<Instruction>(U))
949 if (isa<GetElementPtrInst>(U) ||
953 unsigned OpCode = cast<Instruction>(U)->getOpcode();
954 if ((OpCode == Instruction::Add || OpCode == Instruction::Mul ||
955 OpCode == Instruction::Shl ||
966bool MVEGatherScatterLowering::optimiseOffsets(
Value *Offsets,
BasicBlock *BB,
968 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: trying to optimize: "
969 << *Offsets <<
"\n");
972 if (!isa<Instruction>(Offsets))
994 }
else if (isa<PHINode>(Offs->
getOperand(1))) {
998 bool Changed =
false;
1000 L->contains(cast<Instruction>(Offs->
getOperand(0))))
1001 Changed |= optimiseOffsets(Offs->
getOperand(0), BB, LI);
1003 L->contains(cast<Instruction>(Offs->
getOperand(1))))
1004 Changed |= optimiseOffsets(Offs->
getOperand(1), BB, LI);
1010 }
else if (isa<PHINode>(Offs->
getOperand(1))) {
1019 if (
Phi->getParent() !=
L->getHeader())
1024 Value *Start, *IncrementPerRound;
1026 IncInstruction->
getOpcode() != Instruction::Add)
1029 int IncrementingBlock =
Phi->getIncomingValue(0) == IncInstruction ? 0 : 1;
1034 if (IncrementPerRound->
getType() != OffsSecondOperand->
getType() ||
1035 !
L->isLoopInvariant(OffsSecondOperand))
1041 if (!isa<Constant>(IncrementPerRound) &&
1042 !(isa<Instruction>(IncrementPerRound) &&
1043 !
L->contains(cast<Instruction>(IncrementPerRound))))
1049 if (
Phi->getNumUses() == 2) {
1057 IncrementPerRound,
"LoopIncrement", IncInstruction);
1058 Phi->setIncomingValue(IncrementingBlock, IncInstruction);
1065 NewPhi->
addIncoming(
Phi->getIncomingValue(IncrementingBlock == 1 ? 0 : 1),
1066 Phi->getIncomingBlock(IncrementingBlock == 1 ? 0 : 1));
1069 IncrementPerRound,
"LoopIncrement", IncInstruction);
1071 Phi->getIncomingBlock(IncrementingBlock));
1072 IncrementingBlock = 1;
1080 case Instruction::Add:
1081 case Instruction::Or:
1082 pushOutAdd(NewPhi, OffsSecondOperand, IncrementingBlock == 1 ? 0 : 1);
1084 case Instruction::Mul:
1085 case Instruction::Shl:
1086 pushOutMulShl(Offs->
getOpcode(), NewPhi, IncrementPerRound,
1087 OffsSecondOperand, IncrementingBlock, Builder);
1092 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: simplified loop variable "
1114 if ((Const = dyn_cast<ConstantInt>(NonVectorVal)) &&
1118 if (
N < (
unsigned)(1 << (TargetElemSize - 1))) {
1119 NonVectorVal =
Builder.CreateVectorSplat(
1132 if (XElType && !YElType) {
1133 FixSummands(XElType,
Y);
1134 YElType = cast<FixedVectorType>(
Y->getType());
1135 }
else if (YElType && !XElType) {
1136 FixSummands(YElType,
X);
1137 XElType = cast<FixedVectorType>(
X->getType());
1139 assert(XElType && YElType &&
"Unknown vector types");
1141 if (XElType != YElType) {
1142 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: incompatible gep offsets\n");
1149 Constant *ConstX = dyn_cast<Constant>(
X);
1150 Constant *ConstY = dyn_cast<Constant>(
Y);
1151 if (!ConstX || !ConstY)
1159 if (!ConstXEl || !ConstYEl ||
1162 (
unsigned)(1 << (TargetElemSize - 1)))
1183 Value *&Offsets,
unsigned &Scale,
1185 Value *GEPPtr =
GEP->getPointerOperand();
1187 Scale =
DL->getTypeAllocSize(
GEP->getSourceElementType());
1190 if (
GEP->getNumIndices() != 1 || !isa<Constant>(Offsets))
1194 Value *BaseBasePtr = foldGEP(BaseGEP, Offsets, Scale, Builder);
1198 Offsets, Scale,
GEP->getOperand(1),
1199 DL->getTypeAllocSize(
GEP->getSourceElementType()), Builder);
1200 if (Offsets ==
nullptr)
1208bool MVEGatherScatterLowering::optimiseAddress(
Value *Address,
BasicBlock *BB,
1213 bool Changed =
false;
1214 if (
GEP->hasOneUse() && isa<GetElementPtrInst>(
GEP->getPointerOperand())) {
1217 Builder.SetCurrentDebugLocation(
GEP->getDebugLoc());
1226 assert(Scale == 1 &&
"Expected to fold GEP to a scale of 1");
1228 if (
auto *VecTy = dyn_cast<FixedVectorType>(
Base->getType()))
1234 <<
"\n new : " << *NewAddress <<
"\n");
1235 GEP->replaceAllUsesWith(
1236 Builder.CreateBitCast(NewAddress,
GEP->getType()));
1241 Changed |= optimiseOffsets(
GEP->getOperand(1),
GEP->getParent(), LI);
1245bool MVEGatherScatterLowering::runOnFunction(
Function &
F) {
1248 auto &TPC = getAnalysis<TargetPassConfig>();
1251 if (!
ST->hasMVEIntegerOps())
1253 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1254 DL = &
F.getParent()->getDataLayout();
1258 bool Changed =
false;
1266 isa<FixedVectorType>(II->
getType())) {
1269 }
else if (II && II->
getIntrinsicID() == Intrinsic::masked_scatter &&
1276 for (
unsigned i = 0; i < Gathers.
size(); i++) {
1287 for (
unsigned i = 0; i < Scatters.
size(); i++) {
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static Decomposition decomposeGEP(GEPOperator &GEP, SmallVectorImpl< ConditionTy > &Preconditions, bool IsSigned, const DataLayout &DL)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isAddLikeOr(Instruction *I, const DataLayout &DL)
static bool hasAllGatScatUsers(Instruction *I, const DataLayout &DL)
static bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount)
static Value * CheckAndCreateOffsetAdd(Value *X, unsigned ScaleX, Value *Y, unsigned ScaleY, IRBuilder<> &Builder)
cl::opt< bool > EnableMaskedGatherScatters("enable-arm-maskedgatscat", cl::Hidden, cl::init(true), cl::desc("Enable the generation of masked gathers and scatters"))
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
LLVM Basic Block Representation.
LLVMContext & getContext() const
Get the context in which this basic block lives.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
BinaryOps getOpcode() const
Value * getArgOperand(unsigned i) const
Type * getDestTy() const
Return the destination type, as a convenience.
This is the shared class of boolean and integer constants.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
A parsed version of the target data layout string in and methods for querying it.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Represents a single loop in the control flow graph.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Primary interface to the complete machine description for the target machine.
Target-Independent Code Generator Pass Configuration Options.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
unsigned getNumUses() const
This method computes the number of uses of this Value.
Type * getElementType() const
This class represents zero extension of integer types.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
initializer< Ty > init(const Ty &Val)
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
Pass * createMVEGatherScatterLoweringPass()
bool SimplifyInstructionsInBlock(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
Scan the specified basic block and try to simplify any instructions in it and recursively delete dead...
bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
bool isGatherScatter(IntrinsicInst *IntInst)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool haveNoCommonBitsSet(const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if LHS and RHS have no common bits set.
void initializeMVEGatherScatterLoweringPass(PassRegistry &)
This struct is a compact representation of a valid (non-zero power of two) alignment.