50#define DEBUG_TYPE "scalarizer"
54 cl::desc(
"Allow the scalarizer pass to scalarize "
55 "insertelement/extractelement with variable index"));
62 cl::desc(
"Allow the scalarizer pass to scalarize loads and store"));
71 cl::desc(
"Instruct the scalarizer pass to attempt to keep values of a "
72 "minimum number of bits"));
78 if (isa<PHINode>(Itr))
94using ScatterMap = std::map<std::pair<Value *, Type *>, ValueVector>;
105 unsigned NumPacked = 0;
109 unsigned NumFragments = 0;
112 Type *SplitTy =
nullptr;
116 Type *RemainderTy =
nullptr;
118 Type *getFragmentType(
unsigned I)
const {
119 return RemainderTy &&
I == NumFragments - 1 ? RemainderTy : SplitTy;
127 Scatterer() =
default;
133 const VectorSplit &VS, ValueVector *cachePtr =
nullptr);
136 Value *operator[](
unsigned I);
139 unsigned size()
const {
return VS.NumFragments; }
147 ValueVector *CachePtr;
154 FCmpSplitter(
FCmpInst &fci) : FCI(fci) {}
167 ICmpSplitter(
ICmpInst &ici) : ICI(ici) {}
179struct UnarySplitter {
191struct BinarySplitter {
207 Align getFragmentAlign(
unsigned Frag) {
225 unsigned NumElements =
VS.VecTy->getNumElements();
229 if (
VS.NumPacked > 1) {
232 ExtendMask.
resize(NumElements, -1);
233 for (
unsigned I = 0;
I <
VS.NumPacked; ++
I)
236 InsertMask.
resize(NumElements);
237 for (
unsigned I = 0;
I < NumElements; ++
I)
242 for (
unsigned I = 0;
I <
VS.NumFragments; ++
I) {
243 Value *Fragment = Fragments[
I];
245 unsigned NumPacked =
VS.NumPacked;
246 if (
I ==
VS.NumFragments - 1 &&
VS.RemainderTy) {
247 if (
auto *RemVecTy = dyn_cast<FixedVectorType>(
VS.RemainderTy))
248 NumPacked = RemVecTy->getNumElements();
253 if (NumPacked == 1) {
261 for (
unsigned J = 0; J < NumPacked; ++J)
262 InsertMask[
I *
VS.NumPacked + J] = NumElements + J;
265 for (
unsigned J = 0; J < NumPacked; ++J)
266 InsertMask[
I *
VS.NumPacked + J] =
I *
VS.NumPacked + J;
275T getWithDefaultOverride(
const cl::opt<T> &ClOption,
276 const std::optional<T> &DefaultOverride) {
278 : DefaultOverride.value_or(ClOption);
281class ScalarizerVisitor :
public InstVisitor<ScalarizerVisitor, bool> {
284 : DT(DT), ScalarizeVariableInsertExtract(getWithDefaultOverride(
286 Options.ScalarizeVariableInsertExtract)),
316 void gather(
Instruction *
Op,
const ValueVector &CV,
const VectorSplit &VS);
318 bool canTransferMetadata(
unsigned Kind);
319 void transferMetadataAndIRFlags(
Instruction *
Op,
const ValueVector &CV);
320 std::optional<VectorSplit> getVectorSplit(
Type *Ty);
321 std::optional<VectorLayout> getVectorLayout(
Type *Ty,
Align Alignment,
325 template<
typename T>
bool splitUnary(
Instruction &,
const T &);
326 template<
typename T>
bool splitBinary(
Instruction &,
const T &);
330 ScatterMap Scattered;
338 const bool ScalarizeVariableInsertExtract;
339 const bool ScalarizeLoadStore;
340 const unsigned ScalarizeMinBits;
346 const VectorSplit &VS, ValueVector *cachePtr)
347 : BB(bb), BBI(bbi),
V(
v),
VS(
VS), CachePtr(cachePtr) {
350 Tmp.resize(
VS.NumFragments,
nullptr);
352 assert((CachePtr->empty() ||
VS.NumFragments == CachePtr->size() ||
354 "Inconsistent vector sizes");
355 if (
VS.NumFragments > CachePtr->size())
356 CachePtr->resize(
VS.NumFragments,
nullptr);
361Value *Scatterer::operator[](
unsigned Frag) {
362 ValueVector &CV = CachePtr ? *CachePtr : Tmp;
372 V->getName() +
".i" +
Twine(Frag));
376 Type *FragmentTy =
VS.getFragmentType(Frag);
378 if (
auto *VecTy = dyn_cast<FixedVectorType>(FragmentTy)) {
380 for (
unsigned J = 0; J < VecTy->getNumElements(); ++J)
381 Mask.push_back(Frag *
VS.NumPacked + J);
384 V->getName() +
".i" +
Twine(Frag));
396 unsigned J =
Idx->getZExtValue();
398 if (Frag *
VS.NumPacked == J) {
399 CV[Frag] =
Insert->getOperand(1);
403 if (
VS.NumPacked == 1 && !CV[J]) {
407 CV[J] =
Insert->getOperand(1);
411 V->getName() +
".i" +
Twine(Frag));
417bool ScalarizerVisitor::visit(
Function &
F) {
418 assert(Gathered.empty() && Scattered.empty());
430 if (
Done &&
I->getType()->isVoidTy())
431 I->eraseFromParent();
440 const VectorSplit &VS) {
441 if (
Argument *VArg = dyn_cast<Argument>(V)) {
446 return Scatterer(BB, BB->
begin(), V, VS, &Scattered[{V, VS.SplitTy}]);
455 if (!DT->isReachableFromEntry(VOp->getParent()))
463 &Scattered[{
V,
VS.SplitTy}]);
474void ScalarizerVisitor::gather(
Instruction *
Op,
const ValueVector &CV,
475 const VectorSplit &VS) {
476 transferMetadataAndIRFlags(
Op, CV);
480 ValueVector &SV = Scattered[{
Op,
VS.SplitTy}];
482 for (
unsigned I = 0, E = SV.size();
I != E; ++
I) {
484 if (V ==
nullptr || SV[
I] == CV[
I])
488 if (isa<Instruction>(CV[
I]))
489 CV[
I]->takeName(Old);
491 PotentiallyDeadInstrs.emplace_back(Old);
495 Gathered.push_back(GatherList::value_type(
Op, &SV));
501 Op->replaceAllUsesWith(CV);
502 PotentiallyDeadInstrs.emplace_back(
Op);
509bool ScalarizerVisitor::canTransferMetadata(
unsigned Tag) {
510 return (
Tag == LLVMContext::MD_tbaa
511 ||
Tag == LLVMContext::MD_fpmath
512 ||
Tag == LLVMContext::MD_tbaa_struct
513 ||
Tag == LLVMContext::MD_invariant_load
514 ||
Tag == LLVMContext::MD_alias_scope
515 ||
Tag == LLVMContext::MD_noalias
516 ||
Tag == LLVMContext::MD_mem_parallel_loop_access
517 ||
Tag == LLVMContext::MD_access_group);
522void ScalarizerVisitor::transferMetadataAndIRFlags(
Instruction *
Op,
523 const ValueVector &CV) {
525 Op->getAllMetadataOtherThanDebugLoc(MDs);
526 for (
unsigned I = 0, E = CV.size();
I != E; ++
I) {
528 for (
const auto &MD : MDs)
529 if (canTransferMetadata(MD.first))
530 New->setMetadata(MD.first, MD.second);
531 New->copyIRFlags(
Op);
532 if (
Op->getDebugLoc() && !
New->getDebugLoc())
533 New->setDebugLoc(
Op->getDebugLoc());
539std::optional<VectorSplit> ScalarizerVisitor::getVectorSplit(
Type *Ty) {
541 Split.VecTy = dyn_cast<FixedVectorType>(Ty);
545 unsigned NumElems =
Split.VecTy->getNumElements();
546 Type *ElemTy =
Split.VecTy->getElementType();
551 Split.NumFragments = NumElems;
552 Split.SplitTy = ElemTy;
555 if (
Split.NumPacked >= NumElems)
561 unsigned RemainderElems = NumElems %
Split.NumPacked;
562 if (RemainderElems > 1)
564 else if (RemainderElems == 1)
565 Split.RemainderTy = ElemTy;
574std::optional<VectorLayout>
575ScalarizerVisitor::getVectorLayout(
Type *Ty,
Align Alignment,
577 std::optional<VectorSplit>
VS = getVectorSplit(Ty);
584 if (!
DL.typeSizeEqualsStoreSize(
VS->SplitTy) ||
585 (
VS->RemainderTy && !
DL.typeSizeEqualsStoreSize(
VS->RemainderTy)))
587 Layout.VecAlign = Alignment;
588 Layout.SplitSize =
DL.getTypeStoreSize(
VS->SplitTy);
594template<
typename Splitter>
595bool ScalarizerVisitor::splitUnary(
Instruction &
I,
const Splitter &Split) {
596 std::optional<VectorSplit>
VS = getVectorSplit(
I.getType());
600 std::optional<VectorSplit> OpVS;
601 if (
I.getOperand(0)->getType() ==
I.getType()) {
604 OpVS = getVectorSplit(
I.getOperand(0)->getType());
605 if (!OpVS ||
VS->NumPacked != OpVS->NumPacked)
610 Scatterer
Op = scatter(&
I,
I.getOperand(0), *OpVS);
611 assert(
Op.size() ==
VS->NumFragments &&
"Mismatched unary operation");
613 Res.resize(
VS->NumFragments);
614 for (
unsigned Frag = 0; Frag <
VS->NumFragments; ++Frag)
615 Res[Frag] =
Split(Builder,
Op[Frag],
I.getName() +
".i" +
Twine(Frag));
616 gather(&
I, Res, *VS);
622template<
typename Splitter>
623bool ScalarizerVisitor::splitBinary(
Instruction &
I,
const Splitter &Split) {
624 std::optional<VectorSplit>
VS = getVectorSplit(
I.getType());
628 std::optional<VectorSplit> OpVS;
629 if (
I.getOperand(0)->getType() ==
I.getType()) {
632 OpVS = getVectorSplit(
I.getOperand(0)->getType());
633 if (!OpVS ||
VS->NumPacked != OpVS->NumPacked)
638 Scatterer VOp0 = scatter(&
I,
I.getOperand(0), *OpVS);
639 Scatterer VOp1 = scatter(&
I,
I.getOperand(1), *OpVS);
640 assert(VOp0.size() ==
VS->NumFragments &&
"Mismatched binary operation");
641 assert(VOp1.size() ==
VS->NumFragments &&
"Mismatched binary operation");
643 Res.resize(
VS->NumFragments);
644 for (
unsigned Frag = 0; Frag <
VS->NumFragments; ++Frag) {
645 Value *Op0 = VOp0[Frag];
646 Value *Op1 = VOp1[Frag];
647 Res[Frag] =
Split(Builder, Op0, Op1,
I.getName() +
".i" +
Twine(Frag));
649 gather(&
I, Res, *VS);
659bool ScalarizerVisitor::splitCall(
CallInst &CI) {
660 std::optional<VectorSplit>
VS = getVectorSplit(CI.
getType());
675 ValueVector ScalarOperands(NumArgs);
686 for (
unsigned I = 0;
I != NumArgs; ++
I) {
688 if ([[maybe_unused]]
auto *OpVecTy =
689 dyn_cast<FixedVectorType>(OpI->
getType())) {
690 assert(OpVecTy->getNumElements() ==
VS->VecTy->getNumElements());
691 std::optional<VectorSplit> OpVS = getVectorSplit(OpI->
getType());
692 if (!OpVS || OpVS->NumPacked !=
VS->NumPacked) {
703 Scattered[
I] = scatter(&CI, OpI, *OpVS);
705 OverloadIdx[
I] = Tys.
size();
709 ScalarOperands[
I] = OpI;
715 ValueVector Res(
VS->NumFragments);
716 ValueVector ScalarCallOps(NumArgs);
722 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
723 bool IsRemainder =
I ==
VS->NumFragments - 1 &&
VS->RemainderTy;
724 ScalarCallOps.clear();
727 Tys[0] =
VS->RemainderTy;
729 for (
unsigned J = 0; J != NumArgs; ++J) {
731 ScalarCallOps.push_back(ScalarOperands[J]);
733 ScalarCallOps.push_back(Scattered[J][
I]);
734 if (IsRemainder && OverloadIdx[J] >= 0)
735 Tys[OverloadIdx[J]] = Scattered[J][
I]->getType();
742 Res[
I] = Builder.
CreateCall(NewIntrin, ScalarCallOps,
746 gather(&CI, Res, *VS);
750bool ScalarizerVisitor::visitSelectInst(
SelectInst &SI) {
751 std::optional<VectorSplit>
VS = getVectorSplit(
SI.getType());
755 std::optional<VectorSplit> CondVS;
756 if (isa<FixedVectorType>(
SI.getCondition()->getType())) {
757 CondVS = getVectorSplit(
SI.getCondition()->getType());
758 if (!CondVS || CondVS->NumPacked !=
VS->NumPacked) {
765 Scatterer VOp1 = scatter(&SI,
SI.getOperand(1), *VS);
766 Scatterer VOp2 = scatter(&SI,
SI.getOperand(2), *VS);
767 assert(VOp1.size() ==
VS->NumFragments &&
"Mismatched select");
768 assert(VOp2.size() ==
VS->NumFragments &&
"Mismatched select");
770 Res.resize(
VS->NumFragments);
773 Scatterer VOp0 = scatter(&SI,
SI.getOperand(0), *CondVS);
774 assert(VOp0.size() == CondVS->NumFragments &&
"Mismatched select");
775 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
784 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
791 gather(&SI, Res, *VS);
795bool ScalarizerVisitor::visitICmpInst(
ICmpInst &ICI) {
796 return splitBinary(ICI, ICmpSplitter(ICI));
799bool ScalarizerVisitor::visitFCmpInst(
FCmpInst &FCI) {
800 return splitBinary(FCI, FCmpSplitter(FCI));
803bool ScalarizerVisitor::visitUnaryOperator(
UnaryOperator &UO) {
804 return splitUnary(UO, UnarySplitter(UO));
808 return splitBinary(BO, BinarySplitter(BO));
812 std::optional<VectorSplit>
VS = getVectorSplit(GEPI.
getType());
823 for (
unsigned I = 0;
I < 1 + NumIndices; ++
I) {
826 std::optional<VectorSplit> OpVS = getVectorSplit(VecTy);
827 if (!OpVS || OpVS->NumPacked !=
VS->NumPacked) {
831 ScatterOps[
I] = scatter(&GEPI, GEPI.
getOperand(
I), *OpVS);
838 Res.resize(
VS->NumFragments);
839 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
841 SplitOps.
resize(1 + NumIndices);
842 for (
unsigned J = 0; J < 1 + NumIndices; ++J) {
844 SplitOps[J] = ScalarOps[J];
846 SplitOps[J] = ScatterOps[J][
I];
853 NewGEPI->setIsInBounds();
855 gather(&GEPI, Res, *VS);
859bool ScalarizerVisitor::visitCastInst(
CastInst &CI) {
860 std::optional<VectorSplit> DestVS = getVectorSplit(CI.
getDestTy());
864 std::optional<VectorSplit> SrcVS = getVectorSplit(CI.
getSrcTy());
865 if (!SrcVS || SrcVS->NumPacked != DestVS->NumPacked)
869 Scatterer Op0 = scatter(&CI, CI.
getOperand(0), *SrcVS);
870 assert(Op0.size() == SrcVS->NumFragments &&
"Mismatched cast");
872 Res.resize(DestVS->NumFragments);
873 for (
unsigned I = 0;
I < DestVS->NumFragments; ++
I)
877 gather(&CI, Res, *DestVS);
881bool ScalarizerVisitor::visitBitCastInst(
BitCastInst &BCI) {
882 std::optional<VectorSplit> DstVS = getVectorSplit(BCI.
getDestTy());
883 std::optional<VectorSplit> SrcVS = getVectorSplit(BCI.
getSrcTy());
884 if (!DstVS || !SrcVS || DstVS->RemainderTy || SrcVS->RemainderTy)
887 const bool isPointerTy = DstVS->VecTy->getElementType()->isPointerTy();
893 Scatterer Op0 = scatter(&BCI, BCI.
getOperand(0), *SrcVS);
895 Res.resize(DstVS->NumFragments);
897 unsigned DstSplitBits = DstVS->SplitTy->getPrimitiveSizeInBits();
898 unsigned SrcSplitBits = SrcVS->SplitTy->getPrimitiveSizeInBits();
901 assert(DstVS->NumFragments == SrcVS->NumFragments);
902 for (
unsigned I = 0;
I < DstVS->NumFragments; ++
I) {
906 }
else if (SrcSplitBits % DstSplitBits == 0) {
910 MidVS.NumPacked = DstVS->NumPacked;
911 MidVS.NumFragments = SrcSplitBits / DstSplitBits;
913 MidVS.NumPacked * MidVS.NumFragments);
914 MidVS.SplitTy = DstVS->SplitTy;
917 for (
unsigned I = 0;
I < SrcVS->NumFragments; ++
I) {
923 while ((VI = dyn_cast<Instruction>(V)) &&
924 VI->getOpcode() == Instruction::BitCast)
925 V =
VI->getOperand(0);
929 Scatterer Mid = scatter(&BCI, V, MidVS);
930 for (
unsigned J = 0; J < MidVS.NumFragments; ++J)
931 Res[ResI++] = Mid[J];
933 }
else if (DstSplitBits % SrcSplitBits == 0) {
937 MidVS.NumFragments = DstSplitBits / SrcSplitBits;
938 MidVS.NumPacked = SrcVS->NumPacked;
940 MidVS.NumPacked * MidVS.NumFragments);
941 MidVS.SplitTy = SrcVS->SplitTy;
945 ConcatOps.
resize(MidVS.NumFragments);
946 for (
unsigned I = 0;
I < DstVS->NumFragments; ++
I) {
947 for (
unsigned J = 0; J < MidVS.NumFragments; ++J)
948 ConcatOps[J] = Op0[SrcI++];
949 Value *
V = concatenate(Builder, ConcatOps, MidVS,
958 gather(&BCI, Res, *DstVS);
963 std::optional<VectorSplit>
VS = getVectorSplit(IEI.
getType());
968 Scatterer Op0 = scatter(&IEI, IEI.
getOperand(0), *VS);
973 Res.resize(
VS->NumFragments);
975 if (
auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
976 unsigned Idx = CI->getZExtValue();
977 unsigned Fragment =
Idx /
VS->NumPacked;
978 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
981 if (Fragment ==
VS->NumFragments - 1 &&
VS->RemainderTy &&
982 !
VS->RemainderTy->isVectorTy())
996 if (!ScalarizeVariableInsertExtract ||
VS->NumPacked > 1)
999 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1000 Value *ShouldReplace =
1004 Res[
I] = Builder.
CreateSelect(ShouldReplace, NewElt, OldElt,
1009 gather(&IEI, Res, *VS);
1019 Scatterer Op0 = scatter(&EEI, EEI.
getOperand(0), *VS);
1022 if (
auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
1023 unsigned Idx = CI->getZExtValue();
1024 unsigned Fragment =
Idx /
VS->NumPacked;
1025 Value *Res = Op0[Fragment];
1027 if (Fragment ==
VS->NumFragments - 1 &&
VS->RemainderTy &&
1028 !
VS->RemainderTy->isVectorTy())
1032 replaceUses(&EEI, Res);
1037 if (!ScalarizeVariableInsertExtract ||
VS->NumPacked > 1)
1041 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1042 Value *ShouldExtract =
1049 replaceUses(&EEI, Res);
1054 std::optional<VectorSplit>
VS = getVectorSplit(SVI.
getType());
1055 std::optional<VectorSplit> VSOp =
1057 if (!VS || !VSOp ||
VS->NumPacked > 1 || VSOp->NumPacked > 1)
1060 Scatterer Op0 = scatter(&SVI, SVI.
getOperand(0), *VSOp);
1061 Scatterer Op1 = scatter(&SVI, SVI.
getOperand(1), *VSOp);
1063 Res.resize(
VS->NumFragments);
1065 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1069 else if (
unsigned(Selector) < Op0.size())
1070 Res[
I] = Op0[Selector];
1072 Res[
I] = Op1[Selector - Op0.size()];
1074 gather(&SVI, Res, *VS);
1078bool ScalarizerVisitor::visitPHINode(
PHINode &
PHI) {
1079 std::optional<VectorSplit>
VS = getVectorSplit(
PHI.getType());
1085 Res.resize(
VS->NumFragments);
1087 unsigned NumOps =
PHI.getNumOperands();
1088 for (
unsigned I = 0;
I <
VS->NumFragments; ++
I) {
1093 for (
unsigned I = 0;
I < NumOps; ++
I) {
1094 Scatterer
Op = scatter(&
PHI,
PHI.getIncomingValue(
I), *VS);
1096 for (
unsigned J = 0; J <
VS->NumFragments; ++J)
1097 cast<PHINode>(Res[J])->addIncoming(
Op[J], IncomingBlock);
1099 gather(&
PHI, Res, *VS);
1103bool ScalarizerVisitor::visitLoadInst(
LoadInst &LI) {
1104 if (!ScalarizeLoadStore)
1109 std::optional<VectorLayout> Layout = getVectorLayout(
1117 Res.resize(Layout->VS.NumFragments);
1119 for (
unsigned I = 0;
I < Layout->VS.NumFragments; ++
I) {
1121 Align(Layout->getFragmentAlign(
I)),
1124 gather(&LI, Res, Layout->VS);
1128bool ScalarizerVisitor::visitStoreInst(
StoreInst &SI) {
1129 if (!ScalarizeLoadStore)
1134 Value *FullValue =
SI.getValueOperand();
1135 std::optional<VectorLayout> Layout = getVectorLayout(
1136 FullValue->
getType(),
SI.getAlign(),
SI.getModule()->getDataLayout());
1141 Scatterer VPtr = scatter(&SI,
SI.getPointerOperand(), Layout->VS);
1142 Scatterer VVal = scatter(&SI, FullValue, Layout->VS);
1145 Stores.resize(Layout->VS.NumFragments);
1146 for (
unsigned I = 0;
I < Layout->VS.NumFragments; ++
I) {
1152 transferMetadataAndIRFlags(&SI, Stores);
1156bool ScalarizerVisitor::visitCallInst(
CallInst &CI) {
1157 return splitCall(CI);
1160bool ScalarizerVisitor::visitFreezeInst(
FreezeInst &FI) {
1168bool ScalarizerVisitor::finish() {
1171 if (Gathered.empty() && Scattered.empty() && !Scalarized)
1173 for (
const auto &GMI : Gathered) {
1175 ValueVector &CV = *GMI.second;
1176 if (!
Op->use_empty()) {
1180 if (
auto *Ty = dyn_cast<FixedVectorType>(
Op->getType())) {
1183 if (isa<PHINode>(
Op))
1186 VectorSplit
VS = *getVectorSplit(Ty);
1187 assert(
VS.NumFragments == CV.size());
1189 Res = concatenate(Builder, CV, VS,
Op->getName());
1193 assert(CV.size() == 1 &&
Op->getType() == CV[0]->getType());
1198 Op->replaceAllUsesWith(Res);
1200 PotentiallyDeadInstrs.emplace_back(
Op);
1213 ScalarizerVisitor Impl(DT, Options);
1214 bool Changed = Impl.visit(
F);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Module.h This file contains the declarations for the Module class.
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isTriviallyScalariable(Intrinsic::ID ID)
static cl::opt< bool > ClScalarizeVariableInsertExtract("scalarize-variable-insert-extract", cl::init(true), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize " "insertelement/extractelement with variable index"))
static cl::opt< bool > ClScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize loads and store"))
static cl::opt< unsigned > ClScalarizeMinBits("scalarize-min-bits", cl::init(0), cl::Hidden, cl::desc("Instruct the scalarizer pass to attempt to keep values of a " "minimum number of bits"))
This pass converts vector operations into scalar operations (or, optionally, operations on smaller ve...
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Type * getSrcTy() const
Return the source type, as a convenience.
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Type * getDestTy() const
Return the destination type, as a convenience.
This is the shared class of boolean and integer constants.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
Type * getSourceElementType() const
unsigned getNumIndices() const
This instruction compares its operands according to the predicate given to the constructor.
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateFreeze(Value *V, const Twine &Name="")
Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
Base class for instruction visitors.
RetTy visitFreezeInst(FreezeInst &I)
RetTy visitFCmpInst(FCmpInst &I)
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitShuffleVectorInst(ShuffleVectorInst &I)
RetTy visitBitCastInst(BitCastInst &I)
void visit(Iterator Start, Iterator End)
RetTy visitPHINode(PHINode &I)
RetTy visitUnaryOperator(UnaryOperator &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitBinaryOperator(BinaryOperator &I)
RetTy visitICmpInst(ICmpInst &I)
RetTy visitCallInst(CallInst &I)
RetTy visitCastInst(CastInst &I)
RetTy visitSelectInst(SelectInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
An instruction for reading from memory.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserve()
Mark an analysis as preserved.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
This class represents the LLVM 'select' instruction.
This instruction constructs a fixed permutation of two input vectors.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
VectorType * getType() const
Overload to return most specific vector type.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
int getNumOccurrences() const
self_iterator getIterator()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It)
Advance It while it points to a debug instruction and return the result.
bool isPointerTy(const Type *T)
DWARFExpression::Operation Op
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
This struct is a compact representation of a valid (non-zero power of two) alignment.