46#include "llvm/IR/IntrinsicsHexagon.h"
69#define DEBUG_TYPE "hexagon-lowering"
73 cl::desc(
"Control jump table emission on Hexagon target"));
77 cl::desc(
"Enable Hexagon SDNode scheduling"));
80 cl::desc(
"Enable Fast Math processing"));
84 cl::desc(
"Set minimum jump tables"));
88 cl::desc(
"Max #stores to inline memcpy"));
92 cl::desc(
"Max #stores to inline memcpy"));
96 cl::desc(
"Max #stores to inline memmove"));
101 cl::desc(
"Max #stores to inline memmove"));
105 cl::desc(
"Max #stores to inline memset"));
109 cl::desc(
"Max #stores to inline memset"));
113 cl::desc(
"Rewrite unaligned loads as a pair of aligned loads"));
118 cl::desc(
"Disable minimum alignment of 1 for "
119 "arguments passed by value on stack"));
123 class HexagonCCState :
public CCState {
124 unsigned NumNamedVarArgParams = 0;
129 unsigned NumNamedArgs)
131 NumNamedVarArgParams(NumNamedArgs) {}
132 unsigned getNumNamedVarArgParams()
const {
return NumNamedVarArgParams; }
144 Hexagon::R0, Hexagon::R1, Hexagon::R2,
145 Hexagon::R3, Hexagon::R4, Hexagon::R5
147 const unsigned NumArgRegs = std::size(ArgRegs);
151 if (RegNum != NumArgRegs && RegNum % 2 == 1)
160#include "HexagonGenCallingConv.inc"
179 Chain, dl, Dst, Src, SizeNode,
Flags.getNonZeroByValAlign(),
193 return CCInfo.
CheckReturn(Outs, RetCC_Hexagon_HVX);
223 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
272 .
Case(
"r0", Hexagon::R0)
273 .
Case(
"r1", Hexagon::R1)
274 .
Case(
"r2", Hexagon::R2)
275 .
Case(
"r3", Hexagon::R3)
276 .
Case(
"r4", Hexagon::R4)
277 .
Case(
"r5", Hexagon::R5)
278 .
Case(
"r6", Hexagon::R6)
279 .
Case(
"r7", Hexagon::R7)
280 .
Case(
"r8", Hexagon::R8)
281 .
Case(
"r9", Hexagon::R9)
282 .
Case(
"r10", Hexagon::R10)
283 .
Case(
"r11", Hexagon::R11)
284 .
Case(
"r12", Hexagon::R12)
285 .
Case(
"r13", Hexagon::R13)
286 .
Case(
"r14", Hexagon::R14)
287 .
Case(
"r15", Hexagon::R15)
288 .
Case(
"r16", Hexagon::R16)
289 .
Case(
"r17", Hexagon::R17)
290 .
Case(
"r18", Hexagon::R18)
291 .
Case(
"r19", Hexagon::R19)
292 .
Case(
"r20", Hexagon::R20)
293 .
Case(
"r21", Hexagon::R21)
294 .
Case(
"r22", Hexagon::R22)
295 .
Case(
"r23", Hexagon::R23)
296 .
Case(
"r24", Hexagon::R24)
297 .
Case(
"r25", Hexagon::R25)
298 .
Case(
"r26", Hexagon::R26)
299 .
Case(
"r27", Hexagon::R27)
300 .
Case(
"r28", Hexagon::R28)
301 .
Case(
"r29", Hexagon::R29)
302 .
Case(
"r30", Hexagon::R30)
303 .
Case(
"r31", Hexagon::R31)
304 .
Case(
"r1:0", Hexagon::D0)
305 .
Case(
"r3:2", Hexagon::D1)
306 .
Case(
"r5:4", Hexagon::D2)
307 .
Case(
"r7:6", Hexagon::D3)
308 .
Case(
"r9:8", Hexagon::D4)
309 .
Case(
"r11:10", Hexagon::D5)
310 .
Case(
"r13:12", Hexagon::D6)
311 .
Case(
"r15:14", Hexagon::D7)
312 .
Case(
"r17:16", Hexagon::D8)
313 .
Case(
"r19:18", Hexagon::D9)
314 .
Case(
"r21:20", Hexagon::D10)
315 .
Case(
"r23:22", Hexagon::D11)
316 .
Case(
"r25:24", Hexagon::D12)
317 .
Case(
"r27:26", Hexagon::D13)
318 .
Case(
"r29:28", Hexagon::D14)
319 .
Case(
"r31:30", Hexagon::D15)
320 .
Case(
"sp", Hexagon::R29)
321 .
Case(
"fp", Hexagon::R30)
322 .
Case(
"lr", Hexagon::R31)
323 .
Case(
"p0", Hexagon::P0)
324 .
Case(
"p1", Hexagon::P1)
325 .
Case(
"p2", Hexagon::P2)
326 .
Case(
"p3", Hexagon::P3)
327 .
Case(
"sa0", Hexagon::SA0)
328 .
Case(
"lc0", Hexagon::LC0)
329 .
Case(
"sa1", Hexagon::SA1)
330 .
Case(
"lc1", Hexagon::LC1)
331 .
Case(
"m0", Hexagon::M0)
332 .
Case(
"m1", Hexagon::M1)
333 .
Case(
"usr", Hexagon::USR)
334 .
Case(
"ugp", Hexagon::UGP)
335 .
Case(
"cs0", Hexagon::CS0)
336 .
Case(
"cs1", Hexagon::CS1)
366 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
368 if (RVLocs[i].getValVT() ==
MVT::i1) {
378 Register PredR =
MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);
390 RVLocs[i].getValVT(), Glue);
416 bool IsStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
430 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.
getContext(),
434 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
436 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
438 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
443 IsVarArg, IsStructRet, StructAttrFlag, Outs,
452 :
"Argument must be passed on stack. "
453 "Not eligible for Tail Call\n"));
456 unsigned NumBytes = CCInfo.getNextStackOffset();
464 bool NeedsArgAlign =
false;
465 Align LargestAlignSeen;
467 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
473 NeedsArgAlign |= ArgAlign;
499 StackPtr.getValueType());
502 LargestAlignSeen = std::max(
504 if (
Flags.isByVal()) {
524 if (NeedsArgAlign && Subtarget.
hasV60Ops()) {
525 LLVM_DEBUG(
dbgs() <<
"Function needs byte stack align due to call args\n");
526 Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
527 LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);
532 if (!MemOpChains.
empty())
546 for (
const auto &R : RegsToPass) {
547 Chain = DAG.
getCopyToReg(Chain, dl, R.first, R.second, Glue);
562 for (
const auto &R : RegsToPass) {
563 Chain = DAG.
getCopyToReg(Chain, dl, R.first, R.second, Glue);
578 dyn_cast<ExternalSymbolSDNode>(
Callee)) {
590 for (
const auto &R : RegsToPass)
594 assert(Mask &&
"Missing call preserved mask for calling convention");
611 Chain = DAG.
getNode(OpCode, dl, NodeTys, Ops);
646 Base = Op->getOperand(0);
647 Offset = Op->getOperand(1);
648 if (!isa<ConstantSDNode>(
Offset.getNode()))
652 int32_t V = cast<ConstantSDNode>(
Offset.getNode())->getSExtValue();
661 unsigned LR = HRI.getRARegister();
667 unsigned NumOps = Op.getNumOperands();
668 if (Op.getOperand(NumOps-1).getValueType() ==
MVT::Glue)
672 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
687 for (; NumVals; --NumVals, ++i) {
688 Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
691 HMFI.setHasClobberLR(
true);
707 SDValue Chain = Op.getOperand(0);
722 SDValue Chain = Op.getOperand(0);
730 SDValue Chain = Op.getOperand(0);
731 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
733 if (IntNo == Intrinsic::hexagon_prefetch) {
745 SDValue Chain = Op.getOperand(0);
751 assert(AlignConst &&
"Non-constant Align in LowerDYNAMIC_STACKALLOC");
757 A = HFI.getStackAlign().value();
760 dbgs () << __func__ <<
" Align: " <<
A <<
" Size: ";
761 Size.getNode()->dump(&DAG);
786 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
791 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
793 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
795 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
804 switch (RC.
getID()) {
805 case Hexagon::IntRegsRegClassID:
806 return Reg - Hexagon::R0 + 1;
807 case Hexagon::DoubleRegsRegClassID:
808 return (Reg - Hexagon::D0 + 1) * 2;
809 case Hexagon::HvxVRRegClassID:
810 return Reg - Hexagon::V0 + 1;
811 case Hexagon::HvxWRRegClassID:
812 return (Reg - Hexagon::W0 + 1) * 2;
819 HFL.FirstVarArgSavedReg = 0;
822 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
825 bool ByVal =
Flags.isByVal();
835 (!ByVal || (ByVal &&
Flags.getByValSize() > 8));
864 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.
getLocReg());
870 unsigned ObjSize =
Flags.isByVal()
871 ?
Flags.getByValSize()
879 if (
Flags.isByVal()) {
893 for (
int i = HFL.FirstVarArgSavedReg; i < 6; i++)
894 MRI.addLiveIn(Hexagon::R0+i);
898 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
902 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
903 bool RequiresPadding = (NumVarArgRegs & 1);
904 int RegSaveAreaSizePlusPadding = RequiresPadding
905 ? (NumVarArgRegs + 1) * 4
908 if (RegSaveAreaSizePlusPadding > 0) {
911 if (!(RegAreaStart % 8))
912 RegAreaStart = (RegAreaStart + 7) & -8;
914 int RegSaveAreaFrameIndex =
916 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
919 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
921 HMFI.setVarArgsFrameIndex(FI);
927 HMFI.setRegSavedAreaStartFrameIndex(FI);
928 HMFI.setVarArgsFrameIndex(FI);
937 HMFI.setVarArgsFrameIndex(FI);
950 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
962 SDValue FIN = Op.getOperand(1);
970 SDValue SavedRegAreaStartFrameIndex =
975 if (HFL.FirstVarArgSavedReg & 1)
976 SavedRegAreaStartFrameIndex =
985 SavedRegAreaStartFrameIndex,
1014 SDValue Chain = Op.getOperand(0);
1015 SDValue DestPtr = Op.getOperand(1);
1016 SDValue SrcPtr = Op.getOperand(2);
1017 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
1018 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
1024 false,
false,
false,
1029 const SDLoc &dl(Op);
1053 auto isSExtFree = [
this](
SDValue N) {
1054 switch (
N.getOpcode()) {
1060 EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();
1066 return ThisBW >= OrigBW;
1077 bool IsNegative =
C &&
C->getAPIntValue().isNegative();
1078 if (IsNegative || isSExtFree(
LHS) || isSExtFree(
RHS))
1089 SDValue PredOp = Op.getOperand(0);
1090 SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
1092 const SDLoc &dl(Op);
1112 EVT ValTy = Op.getValueType();
1115 bool isVTi1Type =
false;
1116 if (
auto *CV = dyn_cast<ConstantVector>(CPN->
getConstVal())) {
1117 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1120 unsigned VecLen = CV->getNumOperands();
1122 "conversion only supported for pow2 VectorSize");
1123 for (
unsigned i = 0; i < VecLen; ++i)
1139 else if (isVTi1Type)
1145 assert(cast<ConstantPoolSDNode>(
T)->getTargetFlags() == TF &&
1146 "Inconsistent target flag encountered");
1148 if (IsPositionIndependent)
1155 EVT VT = Op.getValueType();
1156 int Idx = cast<JumpTableSDNode>(Op)->getIndex();
1176 EVT VT = Op.getValueType();
1178 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1198 EVT VT = Op.getValueType();
1200 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1218 auto *GAN = cast<GlobalAddressSDNode>(Op);
1220 auto *GV = GAN->getGlobal();
1221 int64_t
Offset = GAN->getOffset();
1229 if (GO && Subtarget.
useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1251 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
1277 unsigned char OperandFlags)
const {
1296 assert(Mask &&
"Missing call preserved mask for calling convention");
1331 if (IsPositionIndependent) {
1403 Hexagon::R0,
Flags);
1539 for (
unsigned LegalIntOp :
1579 for (
unsigned IntExpOp :
1588 for (
unsigned FPExpOp :
1624 static const unsigned VectExpOps[] = {
1650 for (
unsigned VectExpOp : VectExpOps)
1664 if (VT.getVectorElementType() !=
MVT::i32) {
1704 if (NativeVT.getVectorElementType() !=
MVT::i1)
1816 initializeHVXLowering();
1836 setLibcallName(RTLIB::FPTOUINT_F32_I128,
"__hexagon_fixunssfti");
1837 setLibcallName(RTLIB::FPTOUINT_F64_I128,
"__hexagon_fixunsdfti");
1936HexagonTargetLowering::validateConstPtrAlignment(
SDValue Ptr,
Align NeedAlign,
1938 auto *CA = dyn_cast<ConstantSDNode>(
Ptr);
1941 unsigned Addr = CA->getZExtValue();
1944 if (HaveAlign >= NeedAlign)
1950 DiagnosticInfoMisalignedTrap(
StringRef M)
1956 return DI->
getKind() == DK_MisalignedTrap;
1964 <<
" has alignment " << HaveAlign.
value()
1965 <<
", but the memory access requires " << NeedAlign.
value();
1968 O <<
". The instruction has been replaced with a trap.";
1977 const SDLoc &dl(Op);
1978 auto *
LS = cast<LSBaseSDNode>(
Op.getNode());
1979 assert(!
LS->isIndexed() &&
"Not expecting indexed ops on constant address");
1991 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1992 return (
ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1993 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1994 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1995 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1996 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1997 ID == Intrinsic::hexagon_L2_loadrub_pbr);
2006 V = cast<Operator>(V)->getOperand(0);
2008 V = cast<Instruction>(V)->getOperand(0);
2021 if (Blk == Parent) {
2026 BaseVal = BackEdgeVal;
2028 }
while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2031 if (IntrBaseVal == BackEdgeVal)
2038 assert(
Idx >= 0 &&
"Unexpected index to incoming argument in PHI");
2046 Value *IntrBaseVal = V;
2053 }
while (BaseVal != V);
2056 if (
const PHINode *PN = dyn_cast<PHINode>(V))
2070 unsigned Intrinsic)
const {
2071 switch (Intrinsic) {
2072 case Intrinsic::hexagon_L2_loadrd_pbr:
2073 case Intrinsic::hexagon_L2_loadri_pbr:
2074 case Intrinsic::hexagon_L2_loadrh_pbr:
2075 case Intrinsic::hexagon_L2_loadruh_pbr:
2076 case Intrinsic::hexagon_L2_loadrb_pbr:
2077 case Intrinsic::hexagon_L2_loadrub_pbr: {
2079 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
2080 auto &Cont =
I.getCalledFunction()->getParent()->getContext();
2084 Type *ElTy =
I.getCalledFunction()->getReturnType()->getStructElementType(0);
2091 Info.align =
DL.getABITypeAlign(
Info.memVT.getTypeForEVT(Cont));
2095 case Intrinsic::hexagon_V6_vgathermw:
2096 case Intrinsic::hexagon_V6_vgathermw_128B:
2097 case Intrinsic::hexagon_V6_vgathermh:
2098 case Intrinsic::hexagon_V6_vgathermh_128B:
2099 case Intrinsic::hexagon_V6_vgathermhw:
2100 case Intrinsic::hexagon_V6_vgathermhw_128B:
2101 case Intrinsic::hexagon_V6_vgathermwq:
2102 case Intrinsic::hexagon_V6_vgathermwq_128B:
2103 case Intrinsic::hexagon_V6_vgathermhq:
2104 case Intrinsic::hexagon_V6_vgathermhq_128B:
2105 case Intrinsic::hexagon_V6_vgathermhwq:
2106 case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2107 const Module &M = *
I.getParent()->getParent()->getParent();
2109 Type *VecTy =
I.getArgOperand(1)->getType();
2111 Info.ptrVal =
I.getArgOperand(0);
2114 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2127 return X.getValueType().isScalarInteger();
2147 unsigned DefinedValues)
const {
2152 unsigned Index)
const {
2184 unsigned Action = getPreferredHvxVectorAction(VT);
2204 unsigned Action = getCustomHvxOperationAction(Op);
2211std::pair<SDValue, int>
2212HexagonTargetLowering::getBaseAndOffset(
SDValue Addr)
const {
2215 if (
auto *CN = dyn_cast<const ConstantSDNode>(Op1.
getNode()))
2216 return {
Addr.getOperand(0), CN->getSExtValue() };
2226 const auto *SVN = cast<ShuffleVectorSDNode>(Op);
2228 assert(AM.
size() <= 8 &&
"Unexpected shuffle mask");
2229 unsigned VecLen = AM.
size();
2233 "HVX shuffles should be legal");
2236 SDValue Op0 = Op.getOperand(0);
2237 SDValue Op1 = Op.getOperand(1);
2238 const SDLoc &dl(Op);
2243 if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2252 if (AM[
F] >=
int(VecLen)) {
2260 for (
int M : Mask) {
2262 for (
unsigned j = 0; j != ElemBytes; ++j)
2265 for (
unsigned j = 0; j != ElemBytes; ++j)
2278 for (
unsigned i = 0, e = ByteMask.
size(); i != e; ++i) {
2286 if (ByteMask.
size() == 4) {
2288 if (MaskIdx == (0x03020100 | MaskUnd))
2291 if (MaskIdx == (0x00010203 | MaskUnd)) {
2299 getCombine(Op1, Op0, dl, typeJoin({ty(Op1), ty(Op0)}), DAG);
2300 if (MaskIdx == (0x06040200 | MaskUnd))
2301 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2302 if (MaskIdx == (0x07050301 | MaskUnd))
2303 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2306 getCombine(Op0, Op1, dl, typeJoin({ty(Op0), ty(Op1)}), DAG);
2307 if (MaskIdx == (0x02000604 | MaskUnd))
2308 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2309 if (MaskIdx == (0x03010705 | MaskUnd))
2310 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2313 if (ByteMask.
size() == 8) {
2315 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2318 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2325 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2326 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2327 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2328 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2329 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2330 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2331 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2332 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2333 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2334 VectorPair
P = opSplit(Op0, dl, DAG);
2335 return getInstr(Hexagon::S2_packhl, dl, VecTy, {
P.second,
P.first}, DAG);
2339 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2340 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2341 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2342 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2350 switch (Op.getOpcode()) {
2352 if (
SDValue S = cast<BuildVectorSDNode>(Op)->getSplatValue())
2356 return Op.getOperand(0);
2366 switch (
Op.getOpcode()) {
2380 if (
SDValue Sp = getSplatValue(
Op.getOperand(1), DAG))
2381 return DAG.
getNode(NewOpc,
SDLoc(Op), ty(Op),
Op.getOperand(0), Sp);
2387 const SDLoc &dl(Op);
2393 if (
SDValue S = getVectorShiftByInt(Op, DAG))
2407 MVT ResTy = ty(Res);
2415 auto ShiftPartI8 = [&dl, &DAG,
this](
unsigned Opc,
SDValue V,
SDValue A) {
2425 return ShiftPartI8(Opc, Val, Amt);
2427 auto [LoV, HiV] = opSplit(Val, dl, DAG);
2429 {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2434 if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))
2442 SDValue InpV = Op.getOperand(0);
2443 MVT InpTy = ty(InpV);
2445 const SDLoc &dl(Op);
2452 return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);
2467 bool AllConst =
true;
2469 for (
unsigned i = 0, e = Values.
size(); i != e; ++i) {
2476 if (
auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {
2477 const ConstantInt *CI = CN->getConstantIntValue();
2479 }
else if (
auto *CN = dyn_cast<ConstantFPSDNode>(
V.getNode())) {
2480 const ConstantFP *CF = CN->getConstantFPValue();
2497 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2501 if (!isUndef(Elem[First]))
2509 return getZero(dl, VecTy, DAG);
2516 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2517 Consts[1]->getZExtValue() << 16;
2535 int32_t
V = (Consts[0]->getZExtValue() & 0xFF) |
2536 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2537 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2538 Consts[3]->getZExtValue() << 24;
2543 bool IsSplat =
true;
2544 for (
unsigned i = First+1; i != Num; ++i) {
2545 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2561 for (
unsigned i = 0; i != 4; ++i) {
2576 dbgs() <<
"VecTy: " << VecTy <<
'\n';
2588 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2592 if (!isUndef(Elem[First]))
2600 return getZero(dl, VecTy, DAG);
2604 bool IsSplat =
true;
2605 for (
unsigned i = First+1; i != Num; ++i) {
2606 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2625 for (
unsigned i = 0; i != Num; ++i)
2626 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() &
Mask);
2635 : buildVector32(Elem.
take_front(Num/2), dl, HalfTy, DAG);
2638 : buildVector32(Elem.
drop_front(Num/2), dl, HalfTy, DAG);
2639 return getCombine(
H, L, dl, VecTy, DAG);
2646 MVT VecTy = ty(VecV);
2650 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2655 assert((VecWidth % ElemWidth) == 0);
2656 assert(VecWidth == 32 || VecWidth == 64);
2659 MVT ScalarTy = tyScalar(VecTy);
2665 if (
auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {
2666 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2667 if (VecWidth == 64 && ValWidth == 32) {
2668 assert(Off == 0 || Off == 32);
2669 ExtV =
Off == 0 ? LoHalf(VecV, DAG) : HiHalf(VecV, DAG);
2670 }
else if (Off == 0 && (ValWidth % 8) == 0) {
2677 {VecV, WidthV, OffV});
2685 {VecV, WidthV, OffV});
2695HexagonTargetLowering::extractVectorPred(
SDValue VecV,
SDValue IdxV,
2700 MVT VecTy = ty(VecV);
2704 "Vector elements should equal vector width size");
2705 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2708 if (
auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {
2717 if (ValWidth == 1) {
2728 unsigned Scale = VecWidth / ValWidth;
2733 unsigned VecRep = 8 / VecWidth;
2741 T1 = LoHalf(T1, DAG);
2742 T1 = expandPredicate(T1, dl, DAG);
2753 MVT VecTy = ty(VecV);
2755 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2759 assert(VecWidth == 32 || VecWidth == 64);
2760 assert((VecWidth % ValWidth) == 0);
2776 unsigned W =
C->getZExtValue() * ValWidth;
2779 {VecV, ValV, WidthV, OffV});
2785 {VecV, ValV, WidthV, OffV});
2792HexagonTargetLowering::insertVectorPred(
SDValue VecV,
SDValue ValV,
2795 MVT VecTy = ty(VecV);
2799 SDValue ToReg = getInstr(Hexagon::C2_tfrpr, dl,
MVT::i32, {VecV}, DAG);
2805 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {
Ins}, DAG);
2816 for (
unsigned R = Scale;
R > 1;
R /= 2) {
2817 ValR = contractPredicate(ValR, dl, DAG);
2830HexagonTargetLowering::expandPredicate(
SDValue Vec32,
const SDLoc &dl,
2832 assert(ty(Vec32).getSizeInBits() == 32);
2841HexagonTargetLowering::contractPredicate(
SDValue Vec64,
const SDLoc &dl,
2843 assert(ty(Vec64).getSizeInBits() == 64);
2849 {0, 2, 4, 6, 1, 3, 5, 7});
2874 MVT ValTy = ty(Val);
2879 if (ValLen == ResLen)
2882 const SDLoc &dl(Val);
2884 assert(ResLen % ValLen == 0);
2887 for (
unsigned i = 1, e = ResLen / ValLen; i <
e; ++i)
2896 MVT ElemTy = ty(
Hi);
2919 const SDLoc &dl(Op);
2921 for (
unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2925 return buildVector32(Ops, dl, VecTy, DAG);
2927 return buildVector64(Ops, dl, VecTy, DAG);
2931 bool All0 =
true, All1 =
true;
2933 auto *CN = dyn_cast<ConstantSDNode>(
P.getNode());
2934 if (CN ==
nullptr) {
2935 All0 = All1 =
false;
2954 for (
unsigned i = 0; i != 8; ++i) {
2959 for (
unsigned i = 0, e =
A.size()/2; i != e; ++i)
2963 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2973 const SDLoc &dl(Op);
2975 assert(Op.getNumOperands() == 2);
2976 return getCombine(Op.getOperand(1), Op.getOperand(0), dl, VecTy, DAG);
2982 MVT OpTy = ty(Op.getOperand(0));
2986 assert(Scale == Op.getNumOperands() && Scale > 1);
2995 for (
SDValue P : Op.getNode()->op_values()) {
2997 for (
unsigned R = Scale; R > 1; R /= 2) {
2998 W = contractPredicate(W, dl, DAG);
3007 Words[IdxW ^ 1].
clear();
3009 for (
unsigned i = 0, e = Words[IdxW].
size(); i != e; i += 2) {
3010 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
3013 {W0, W1, WidthV, WidthV});
3021 assert(Scale == 2 && Words[IdxW].
size() == 2);
3023 SDValue WW = getCombine(Words[IdxW][1], Words[IdxW][0], dl,
MVT::i64, DAG);
3033 SDValue Vec = Op.getOperand(0);
3035 return extractVector(Vec, Op.getOperand(1),
SDLoc(Op), ElemTy, ty(Op), DAG);
3041 return extractVector(Op.getOperand(0), Op.getOperand(1),
SDLoc(Op),
3042 ty(Op), ty(Op), DAG);
3048 return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
3055 SDValue ValV = Op.getOperand(1);
3056 return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),
3057 SDLoc(Op), ty(ValV), DAG);
3076 const SDLoc &dl(Op);
3077 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
3088 LN = cast<LoadSDNode>(
NL.getNode());
3092 if (!validateConstPtrAlignment(LN->
getBasePtr(), ClaimAlign, dl, DAG))
3093 return replaceMemWithUndef(Op, DAG);
3099 SDValue TP = getInstr(Hexagon::C2_tfrrp, dl, MemTy, {LU}, DAG);
3113 const SDLoc &dl(Op);
3114 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
3127 SN = cast<StoreSDNode>(NS.getNode());
3131 if (!validateConstPtrAlignment(SN->
getBasePtr(), ClaimAlign, dl, DAG))
3132 return replaceMemWithUndef(Op, DAG);
3136 if (ClaimAlign < NeedAlign)
3144 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
3145 MVT LoadTy = ty(Op);
3148 if (HaveAlign >= NeedAlign)
3151 const SDLoc &dl(Op);
3157 bool DoDefault =
false;
3168 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3187 unsigned LoadLen = NeedAlign;
3190 auto BO = getBaseAndOffset(
Base);
3191 unsigned BaseOpc = BO.first.getOpcode();
3195 if (BO.second % LoadLen != 0) {
3198 BO.second -= BO.second % LoadLen;
3213 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen,
Align(LoadLen),
3214 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
3215 MMO->getSuccessOrdering(), MMO->getFailureOrdering());
3231 SDValue X = Op.getOperand(0),
Y = Op.getOperand(1);
3232 auto *CY = dyn_cast<ConstantSDNode>(
Y);
3236 const SDLoc &dl(Op);
3237 SDVTList VTs = Op.getNode()->getVTList();
3240 unsigned Opc = Op.getOpcode();
3244 assert(VY != 0 &&
"This should have been folded");
3268 const SDLoc &dl(Op);
3269 unsigned Opc = Op.getOpcode();
3270 SDValue X = Op.getOperand(0),
Y = Op.getOperand(1),
C = Op.getOperand(2);
3276 EVT CarryTy =
C.getValueType();
3278 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3286 SDValue Chain = Op.getOperand(0);
3288 SDValue Handler = Op.getOperand(2);
3297 unsigned OffsetReg = Hexagon::R28;
3313 unsigned Opc = Op.getOpcode();
3319 if (isHvxOperation(Op.getNode(), DAG)) {
3321 if (
SDValue V = LowerHvxOperation(Op, DAG))
3328 Op.getNode()->dumpr(&DAG);
3330 errs() <<
"Error: check for a non-legal type in this operation\n";
3380 if (isHvxOperation(
N, DAG)) {
3381 LowerHvxOperationWrapper(
N,
Results, DAG);
3387 unsigned Opc =
N->getOpcode();
3392 Results.push_back(opJoin(SplitVectorOp(Op, DAG),
SDLoc(Op), DAG));
3411 if (isHvxOperation(
N, DAG)) {
3412 ReplaceHvxNodeResults(
N,
Results, DAG);
3418 switch (
N->getOpcode()) {
3425 if (
N->getValueType(0) ==
MVT::i8) {
3426 if (
N->getOperand(0).getValueType() ==
MVT::v8i1) {
3428 N->getOperand(0), DAG);
3440 if (isHvxOperation(
N, DCI.
DAG)) {
3441 if (
SDValue V = PerformHvxDAGCombine(
N, DCI))
3447 const SDLoc &dl(Op);
3448 unsigned Opc = Op.getOpcode();
3451 SDValue Op0 = Op.getOperand(0);
3454 EVT TruncTy = Op.getValueType();
3470 switch (
P.getOpcode()) {
3474 return getZero(dl, ty(Op), DCI.
DAG);
3487 Op.getOperand(2), Op.getOperand(1));
3492 SDValue Op0 = Op.getOperand(0);
3495 MVT TruncTy = ty(Op);
3498 if (ty(Elem0) == TruncTy)
3501 if (ty(Elem0).bitsGT(TruncTy))
3507 auto fold0 = [&,
this](
SDValue Op) {
3510 SDValue Shl = Op.getOperand(0);
3511 SDValue Zxt = Op.getOperand(1);
3519 auto *Amt = dyn_cast<ConstantSDNode>(Shl.
getOperand(1));
3520 if (Amt && Amt->getZExtValue() >= 32 && ty(Z).getSizeInBits() <= 32) {
3521 unsigned A = Amt->getZExtValue();
3543 int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3555 if (Constraint.
size() == 1) {
3556 switch (Constraint[0]) {
3571std::pair<unsigned, const TargetRegisterClass*>
3575 if (Constraint.
size() == 1) {
3576 switch (Constraint[0]) {
3580 return {0u,
nullptr};
3586 return {0u, &Hexagon::IntRegsRegClass};
3589 return {0u, &Hexagon::DoubleRegsRegClass};
3594 return {0u,
nullptr};
3595 return {0u, &Hexagon::ModRegsRegClass};
3599 return {0u,
nullptr};
3602 return {0u, &Hexagon::HvxQRRegClass};
3608 return {0u,
nullptr};
3610 return {0u, &Hexagon::HvxVRRegClass};
3613 return {0u, &Hexagon::HvxVRRegClass};
3614 return {0u, &Hexagon::HvxWRRegClass};
3616 return {0u, &Hexagon::HvxWRRegClass};
3620 return {0u,
nullptr};
3631 bool ForCodeSize)
const {
3661 int Scale = AM.
Scale;
3685 return Imm >= -512 && Imm <= 511;
3695 bool IsCalleeStructRet,
3696 bool IsCallerStructRet,
3703 bool CCMatch = CallerCC == CalleeCC;
3711 if (!isa<GlobalAddressSDNode>(
Callee) &&
3712 !isa<ExternalSymbolSDNode>(
Callee)) {
3732 if (IsCalleeStructRet || IsCallerStructRet)
3755 if (Op.size() >= 8 && Op.isAligned(
Align(8)))
3757 if (Op.size() >= 4 && Op.isAligned(
Align(4)))
3759 if (Op.size() >= 2 && Op.isAligned(
Align(2)))
3769 return allowsHvxMemoryAccess(SVT,
Flags,
Fast);
3776 unsigned *
Fast)
const {
3779 return allowsHvxMisalignedMemoryAccesses(SVT,
Flags,
Fast);
3785std::pair<const TargetRegisterClass*, uint8_t>
3793 return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3795 return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3797 return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3809 auto *L = cast<LoadSDNode>(Load);
3810 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3816 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3824 AdjustHvxInstrPostInstrSelection(
MI, Node);
3833 assert((SZ == 32 || SZ == 64) &&
"Only 32/64-bit atomic loads supported");
3834 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3835 : Intrinsic::hexagon_L4_loadd_locked;
3838 auto PtrTy = cast<PointerType>(
Addr->getType());
3840 Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace());
3845 return Builder.CreateBitCast(Call, ValueTy);
3859 assert((SZ == 32 || SZ == 64) &&
"Only 32/64-bit atomic stores supported");
3860 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3861 : Intrinsic::hexagon_S4_stored_locked;
3864 unsigned AS =
Addr->getType()->getPointerAddressSpace();
3866 Val =
Builder.CreateBitCast(Val, CastTy);
3885 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static cl::opt< int > MaxStoresPerMemcpyCL("max-store-memcpy", cl::Hidden, cl::init(6), cl::desc("Max #stores to inline memcpy"))
static Value * getUnderLyingObjectForBrevLdIntr(Value *V)
static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo, ISD::ArgFlagsTy &ArgFlags, CCState &State)
static cl::opt< bool > AlignLoads("hexagon-align-loads", cl::Hidden, cl::init(false), cl::desc("Rewrite unaligned loads as a pair of aligned loads"))
static bool isBrevLdIntrinsic(const Value *Inst)
static cl::opt< int > MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", cl::Hidden, cl::init(4), cl::desc("Max #stores to inline memmove"))
static cl::opt< int > MaxStoresPerMemmoveCL("max-store-memmove", cl::Hidden, cl::init(6), cl::desc("Max #stores to inline memmove"))
static Value * getBrevLdObject(Value *V)
static cl::opt< int > MaxStoresPerMemsetCL("max-store-memset", cl::Hidden, cl::init(8), cl::desc("Max #stores to inline memset"))
static cl::opt< bool > DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden, cl::init(false), cl::desc("Disable minimum alignment of 1 for " "arguments passed by value on stack"))
static Value * returnEdge(const PHINode *PN, Value *IntrBaseVal)
static cl::opt< int > MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", cl::Hidden, cl::init(4), cl::desc("Max #stores to inline memcpy"))
static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl)
CreateCopyOfByValArgument - Make a copy of an aggregate at address specified by "Src" to address "Dst...
static cl::opt< int > MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", cl::Hidden, cl::init(4), cl::desc("Max #stores to inline memset"))
static cl::opt< bool > EmitJumpTables("hexagon-emit-jump-tables", cl::init(true), cl::Hidden, cl::desc("Control jump table emission on Hexagon target"))
static cl::opt< int > MinimumJumpTables("minimum-jump-tables", cl::Hidden, cl::init(5), cl::desc("Set minimum jump tables"))
static cl::opt< bool > EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden, cl::desc("Enable Hexagon SDNode scheduling"))
static cl::opt< bool > EnableFastMath("ffast-math", cl::Hidden, cl::desc("Enable Fast Math processing"))
#define Hexagon_PointerSize
#define HEXAGON_LRFP_SIZE
#define HEXAGON_GOT_SYM_NAME
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
int64_t getSExtValue() const
Get sign extended value.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
size_t size() const
size - Get the array size.
An instruction that atomically checks whether a specified value is in a memory location,...
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
CCValAssign - Represent assignment of one arg/retval to a location.