46 #include "llvm/IR/IntrinsicsHexagon.h"
69 #define DEBUG_TYPE "hexagon-lowering"
73 cl::desc(
"Control jump table emission on Hexagon target"));
77 cl::desc(
"Enable Hexagon SDNode scheduling"));
81 cl::desc(
"Enable Fast Math processing"));
85 cl::desc(
"Set minimum jump tables"));
89 cl::desc(
"Max #stores to inline memcpy"));
93 cl::desc(
"Max #stores to inline memcpy"));
97 cl::desc(
"Max #stores to inline memmove"));
101 cl::desc(
"Max #stores to inline memmove"));
105 cl::desc(
"Max #stores to inline memset"));
109 cl::desc(
"Max #stores to inline memset"));
113 cl::desc(
"Rewrite unaligned loads as a pair of aligned loads"));
118 cl::desc(
"Disable minimum alignment of 1 for "
119 "arguments passed by value on stack"));
123 class HexagonCCState :
public CCState {
124 unsigned NumNamedVarArgParams = 0;
129 unsigned NumNamedArgs)
130 :
CCState(CC, IsVarArg, MF, locs,
C),
131 NumNamedVarArgParams(NumNamedArgs) {}
132 unsigned getNumNamedVarArgParams()
const {
return NumNamedVarArgParams; }
151 if (RegNum != NumArgRegs && RegNum % 2 == 1)
160 #include "HexagonGenCallingConv.inc"
193 return CCInfo.
CheckReturn(Outs, RetCC_Hexagon_HVX);
223 for (
unsigned i = 0;
i != RVLocs.size(); ++
i) {
258 RetOps.push_back(
Flag);
272 .
Case(
"r0", Hexagon::R0)
273 .
Case(
"r1", Hexagon::R1)
275 .
Case(
"r3", Hexagon::R3)
277 .
Case(
"r5", Hexagon::R5)
279 .
Case(
"r7", Hexagon::R7)
280 .
Case(
"r8", Hexagon::R8)
281 .
Case(
"r9", Hexagon::R9)
282 .
Case(
"r10", Hexagon::R10)
283 .
Case(
"r11", Hexagon::R11)
284 .
Case(
"r12", Hexagon::R12)
285 .
Case(
"r13", Hexagon::R13)
286 .
Case(
"r14", Hexagon::R14)
287 .
Case(
"r15", Hexagon::R15)
288 .
Case(
"r16", Hexagon::R16)
289 .
Case(
"r17", Hexagon::R17)
290 .
Case(
"r18", Hexagon::R18)
291 .
Case(
"r19", Hexagon::R19)
292 .
Case(
"r20", Hexagon::R20)
293 .
Case(
"r21", Hexagon::R21)
294 .
Case(
"r22", Hexagon::R22)
295 .
Case(
"r23", Hexagon::R23)
296 .
Case(
"r24", Hexagon::R24)
297 .
Case(
"r25", Hexagon::R25)
298 .
Case(
"r26", Hexagon::R26)
299 .
Case(
"r27", Hexagon::R27)
300 .
Case(
"r28", Hexagon::R28)
301 .
Case(
"r29", Hexagon::R29)
302 .
Case(
"r30", Hexagon::R30)
303 .
Case(
"r31", Hexagon::R31)
304 .
Case(
"r1:0", Hexagon::D0)
305 .
Case(
"r3:2", Hexagon::D1)
306 .
Case(
"r5:4", Hexagon::D2)
307 .
Case(
"r7:6", Hexagon::D3)
308 .
Case(
"r9:8", Hexagon::D4)
309 .
Case(
"r11:10", Hexagon::D5)
310 .
Case(
"r13:12", Hexagon::D6)
311 .
Case(
"r15:14", Hexagon::D7)
312 .
Case(
"r17:16", Hexagon::D8)
313 .
Case(
"r19:18", Hexagon::D9)
314 .
Case(
"r21:20", Hexagon::D10)
315 .
Case(
"r23:22", Hexagon::D11)
316 .
Case(
"r25:24", Hexagon::D12)
317 .
Case(
"r27:26", Hexagon::D13)
318 .
Case(
"r29:28", Hexagon::D14)
319 .
Case(
"r31:30", Hexagon::D15)
320 .
Case(
"sp", Hexagon::R29)
321 .
Case(
"fp", Hexagon::R30)
322 .
Case(
"lr", Hexagon::R31)
323 .
Case(
"p0", Hexagon::P0)
324 .
Case(
"p1", Hexagon::P1)
326 .
Case(
"p3", Hexagon::P3)
327 .
Case(
"sa0", Hexagon::SA0)
329 .
Case(
"sa1", Hexagon::SA1)
330 .
Case(
"lc1", Hexagon::LC1)
333 .
Case(
"usr", Hexagon::USR)
334 .
Case(
"ugp", Hexagon::UGP)
335 .
Case(
"cs0", Hexagon::CS0)
336 .
Case(
"cs1", Hexagon::CS1)
366 for (
unsigned i = 0;
i != RVLocs.size(); ++
i) {
368 if (RVLocs[
i].getValVT() ==
MVT::i1) {
390 RVLocs[
i].getValVT(), Glue);
394 InVals.push_back(RetVal.
getValue(0));
416 bool IsStructRet = Outs.empty() ?
false : Outs[0].Flags.isSRet();
430 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.
getContext(),
434 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
436 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
438 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
443 IsVarArg, IsStructRet, StructAttrFlag, Outs,
452 :
"Argument must be passed on stack. "
453 "Not eligible for Tail Call\n"));
456 unsigned NumBytes = CCInfo.getNextStackOffset();
464 bool NeedsArgAlign =
false;
465 Align LargestAlignSeen;
467 for (
unsigned i = 0,
e = ArgLocs.size();
i !=
e; ++
i) {
473 NeedsArgAlign |= ArgAlign;
499 StackPtr.getValueType());
513 MemOpChains.push_back(
S);
521 RegsToPass.push_back(std::make_pair(VA.
getLocReg(),
Arg));
524 if (NeedsArgAlign && Subtarget.
hasV60Ops()) {
525 LLVM_DEBUG(
dbgs() <<
"Function needs byte stack align due to call args\n");
526 Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
527 LargestAlignSeen =
std::max(LargestAlignSeen, VecAlign);
532 if (!MemOpChains.empty())
546 for (
const auto &R : RegsToPass) {
547 Chain = DAG.
getCopyToReg(Chain, dl, R.first, R.second, Glue);
562 for (
const auto &R : RegsToPass) {
563 Chain = DAG.
getCopyToReg(Chain, dl, R.first, R.second, Glue);
578 dyn_cast<ExternalSymbolSDNode>(
Callee)) {
585 Ops.push_back(Chain);
590 for (
const auto &R : RegsToPass)
591 Ops.push_back(DAG.
getRegister(R.first, R.second.getValueType()));
594 assert(
Mask &&
"Missing call preserved mask for calling convention");
648 Offset =
Op->getOperand(1);
649 if (!isa<ConstantSDNode>(Offset.getNode()))
653 int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
668 unsigned NumOps =
Op.getNumOperands();
669 if (
Op.getOperand(NumOps-1).getValueType() ==
MVT::Glue)
673 unsigned Flags = cast<ConstantSDNode>(
Op.getOperand(
i))->getZExtValue();
688 for (; NumVals; --NumVals, ++
i) {
692 HMFI.setHasClobberLR(
true);
732 unsigned IntNo = cast<ConstantSDNode>(
Op.getOperand(1))->getZExtValue();
734 if (IntNo == Intrinsic::hexagon_prefetch) {
752 assert(AlignConst &&
"Non-constant Align in LowerDYNAMIC_STACKALLOC");
758 A = HFI.getStackAlign().value();
761 dbgs () << __func__ <<
" Align: " << A <<
" Size: ";
762 Size.getNode()->dump(&DAG);
787 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
792 CCInfo.AnalyzeFormalArguments(
Ins, CC_Hexagon_HVX);
794 CCInfo.AnalyzeFormalArguments(
Ins, CC_Hexagon_Legacy);
796 CCInfo.AnalyzeFormalArguments(
Ins, CC_Hexagon);
805 switch (RC.
getID()) {
806 case Hexagon::IntRegsRegClassID:
807 return Reg - Hexagon::R0 + 1;
808 case Hexagon::DoubleRegsRegClassID:
809 return (
Reg - Hexagon::D0 + 1) * 2;
810 case Hexagon::HvxVRRegClassID:
811 return Reg - Hexagon::V0 + 1;
812 case Hexagon::HvxWRRegClassID:
813 return (
Reg - Hexagon::W0 + 1) * 2;
820 HFL.FirstVarArgSavedReg = 0;
823 for (
unsigned i = 0,
e = ArgLocs.size();
i !=
e; ++
i) {
863 InVals.push_back(Copy);
865 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.
getLocReg());
871 unsigned ObjSize = Flags.
isByVal()
884 InVals.push_back(FIN);
894 for (
int i = HFL.FirstVarArgSavedReg;
i < 6;
i++)
899 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
903 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
904 bool RequiresPadding = (NumVarArgRegs & 1);
905 int RegSaveAreaSizePlusPadding = RequiresPadding
906 ? (NumVarArgRegs + 1) * 4
909 if (RegSaveAreaSizePlusPadding > 0) {
912 if (!(RegAreaStart % 8))
913 RegAreaStart = (RegAreaStart + 7) & -8;
915 int RegSaveAreaFrameIndex =
917 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
920 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
922 HMFI.setVarArgsFrameIndex(FI);
928 HMFI.setRegSavedAreaStartFrameIndex(FI);
929 HMFI.setVarArgsFrameIndex(FI);
938 HMFI.setVarArgsFrameIndex(FI);
951 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
971 SDValue SavedRegAreaStartFrameIndex =
976 if (HFL.FirstVarArgSavedReg & 1)
977 SavedRegAreaStartFrameIndex =
986 SavedRegAreaStartFrameIndex,
988 MemOps.push_back(
Store);
997 MemOps.push_back(
Store);
1006 MemOps.push_back(
Store);
1018 const Value *DestSV = cast<SrcValueSDNode>(
Op.getOperand(3))->getValue();
1019 const Value *SrcSV = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
1025 false,
false,
false,
1054 auto isSExtFree = [
this](
SDValue N) {
1055 switch (
N.getOpcode()) {
1061 EVT OrigTy = cast<VTSDNode>(
Op.getOperand(1))->getVT();
1067 return ThisBW >= OrigBW;
1078 bool IsNegative =
C &&
C->getAPIntValue().isNegative();
1079 if (IsNegative || isSExtFree(
LHS) || isSExtFree(
RHS))
1091 SDValue Op1 =
Op.getOperand(1), Op2 =
Op.getOperand(2);
1113 EVT ValTy =
Op.getValueType();
1116 bool isVTi1Type =
false;
1117 if (
auto *CV = dyn_cast<ConstantVector>(CPN->
getConstVal())) {
1118 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1121 unsigned VecLen = CV->getNumOperands();
1123 "conversion only supported for pow2 VectorSize");
1124 for (
unsigned i = 0;
i < VecLen; ++
i)
1125 NewConst.push_back(IRB.
getInt8(CV->getOperand(
i)->isZeroValue()));
1135 unsigned Offset = 0;
1140 else if (isVTi1Type)
1146 assert(cast<ConstantPoolSDNode>(
T)->getTargetFlags() == TF &&
1147 "Inconsistent target flag encountered");
1149 if (IsPositionIndependent)
1156 EVT VT =
Op.getValueType();
1157 int Idx = cast<JumpTableSDNode>(
Op)->getIndex();
1177 EVT VT =
Op.getValueType();
1179 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
1199 EVT VT =
Op.getValueType();
1201 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
1219 auto *GAN = cast<GlobalAddressSDNode>(
Op);
1221 auto *GV = GAN->getGlobal();
1222 int64_t Offset = GAN->getOffset();
1230 if (GO && Subtarget.
useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1252 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
1297 assert(
Mask &&
"Missing call preserved mask for calling convention");
1332 if (IsPositionIndependent) {
1405 Hexagon::R0, Flags);
1541 for (
unsigned LegalIntOp :
1581 for (
unsigned IntExpOp :
1590 for (
unsigned FPExpOp :
1626 static const unsigned VectExpOps[] = {
1652 for (
unsigned VectExpOp : VectExpOps)
1666 if (VT.getVectorElementType() !=
MVT::i32) {
1706 if (NativeVT.getVectorElementType() !=
MVT::i1)
1816 initializeHVXLowering();
1836 setLibcallName(RTLIB::FPTOUINT_F32_I128,
"__hexagon_fixunssfti");
1837 setLibcallName(RTLIB::FPTOUINT_F64_I128,
"__hexagon_fixunsdfti");
1930 HexagonTargetLowering::validateConstPtrAlignment(
SDValue Ptr,
Align NeedAlign,
1932 auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1935 unsigned Addr = CA->getZExtValue();
1938 if (HaveAlign >= NeedAlign)
1944 DiagnosticInfoMisalignedTrap(
StringRef M)
1950 return DI->
getKind() == DK_MisalignedTrap;
1958 <<
" has alignment " << HaveAlign.
value()
1959 <<
", but the memory access requires " << NeedAlign.
value();
1962 O <<
". The instruction has been replaced with a trap.";
1972 auto *
LS = cast<LSBaseSDNode>(
Op.getNode());
1973 assert(!
LS->isIndexed() &&
"Not expecting indexed ops on constant address");
1985 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1986 return (
ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1987 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1988 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1989 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1990 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1991 ID == Intrinsic::hexagon_L2_loadrub_pbr);
2000 V = cast<Operator>(V)->getOperand(0);
2002 V = cast<Instruction>(V)->getOperand(0);
2015 if (Blk == Parent) {
2020 BaseVal = BackEdgeVal;
2022 }
while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2025 if (IntrBaseVal == BackEdgeVal)
2032 assert(Idx >= 0 &&
"Unexpected index to incoming argument in PHI");
2040 Value *IntrBaseVal = V;
2047 }
while (BaseVal != V);
2050 if (
const PHINode *PN = dyn_cast<PHINode>(V))
2064 unsigned Intrinsic)
const {
2065 switch (Intrinsic) {
2066 case Intrinsic::hexagon_L2_loadrd_pbr:
2067 case Intrinsic::hexagon_L2_loadri_pbr:
2068 case Intrinsic::hexagon_L2_loadrh_pbr:
2069 case Intrinsic::hexagon_L2_loadruh_pbr:
2070 case Intrinsic::hexagon_L2_loadrb_pbr:
2071 case Intrinsic::hexagon_L2_loadrub_pbr: {
2073 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
2074 auto &Cont =
I.getCalledFunction()->getParent()->getContext();
2078 Type *ElTy =
I.getCalledFunction()->getReturnType()->getStructElementType(0);
2085 Info.align =
DL.getABITypeAlign(
Info.memVT.getTypeForEVT(Cont));
2089 case Intrinsic::hexagon_V6_vgathermw:
2090 case Intrinsic::hexagon_V6_vgathermw_128B:
2091 case Intrinsic::hexagon_V6_vgathermh:
2092 case Intrinsic::hexagon_V6_vgathermh_128B:
2093 case Intrinsic::hexagon_V6_vgathermhw:
2094 case Intrinsic::hexagon_V6_vgathermhw_128B:
2095 case Intrinsic::hexagon_V6_vgathermwq:
2096 case Intrinsic::hexagon_V6_vgathermwq_128B:
2097 case Intrinsic::hexagon_V6_vgathermhq:
2098 case Intrinsic::hexagon_V6_vgathermhq_128B:
2099 case Intrinsic::hexagon_V6_vgathermhwq:
2100 case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2101 const Module &
M = *
I.getParent()->getParent()->getParent();
2103 Type *VecTy =
I.getArgOperand(1)->getType();
2105 Info.ptrVal =
I.getArgOperand(0);
2108 MaybeAlign(
M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2121 return X.getValueType().isScalarInteger();
2141 unsigned DefinedValues)
const {
2159 unsigned Action = getPreferredHvxVectorAction(VT);
2171 std::pair<SDValue, int>
2172 HexagonTargetLowering::getBaseAndOffset(
SDValue Addr)
const {
2175 if (
auto *CN = dyn_cast<const ConstantSDNode>(Op1.
getNode()))
2176 return {
Addr.getOperand(0), CN->getSExtValue() };
2186 const auto *SVN = cast<ShuffleVectorSDNode>(
Op);
2188 assert(AM.
size() <= 8 &&
"Unexpected shuffle mask");
2189 unsigned VecLen = AM.
size();
2193 "HVX shuffles should be legal");
2203 if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2212 if (AM[
F] >=
int(VecLen)) {
2220 for (
int M :
Mask) {
2222 for (
unsigned j = 0;
j != ElemBytes; ++
j)
2223 ByteMask.push_back(-1);
2225 for (
unsigned j = 0;
j != ElemBytes; ++
j)
2226 ByteMask.push_back(
M*ElemBytes +
j);
2229 assert(ByteMask.size() <= 8);
2238 for (
unsigned i = 0,
e = ByteMask.size();
i !=
e; ++
i) {
2246 if (ByteMask.size() == 4) {
2248 if (MaskIdx == (0x03020100 | MaskUnd))
2251 if (MaskIdx == (0x00010203 | MaskUnd)) {
2259 typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0});
2260 if (MaskIdx == (0x06040200 | MaskUnd))
2261 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2262 if (MaskIdx == (0x07050301 | MaskUnd))
2263 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2266 typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1});
2267 if (MaskIdx == (0x02000604 | MaskUnd))
2268 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2269 if (MaskIdx == (0x03010705 | MaskUnd))
2270 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2273 if (ByteMask.size() == 8) {
2275 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2278 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2285 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2286 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2287 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2288 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2289 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2290 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2291 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2292 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2293 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2294 VectorPair
P = opSplit(Op0, dl, DAG);
2295 return getInstr(Hexagon::S2_packhl, dl, VecTy, {
P.second,
P.first}, DAG);
2299 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2300 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2301 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2302 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2313 switch (
Op.getOpcode()) {
2334 return DAG.
getNode(NewOpc, dl, ty(
Op), Op0,
S);
2344 return getVectorShiftByInt(
Op, DAG);
2349 if (isa<ConstantSDNode>(
Op.getOperand(1).getNode()))
2358 MVT InpTy = ty(InpV);
2367 return getInstr(Hexagon::C2_tfrrp, dl, ResTy,
Ext, DAG);
2382 bool AllConst =
true;
2384 for (
unsigned i = 0,
e = Values.
size();
i !=
e; ++
i) {
2391 if (
auto *CN = dyn_cast<ConstantSDNode>(V.
getNode())) {
2392 const ConstantInt *CI = CN->getConstantIntValue();
2394 }
else if (
auto *CN = dyn_cast<ConstantFPSDNode>(V.
getNode())) {
2395 const ConstantFP *CF = CN->getConstantFPValue();
2412 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2416 if (!isUndef(Elem[First]))
2424 return getZero(dl, VecTy, DAG);
2431 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2432 Consts[1]->getZExtValue() << 16;
2450 int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2451 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2452 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2453 Consts[3]->getZExtValue() << 24;
2458 bool IsSplat =
true;
2459 for (
unsigned i = First+1;
i != Num; ++
i) {
2460 if (Elem[
i] == Elem[First] || isUndef(Elem[
i]))
2476 for (
unsigned i = 0;
i != 4; ++
i) {
2503 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2507 if (!isUndef(Elem[First]))
2515 return getZero(dl, VecTy, DAG);
2519 bool IsSplat =
true;
2520 for (
unsigned i = First+1;
i != Num; ++
i) {
2521 if (Elem[
i] == Elem[First] || isUndef(Elem[
i]))
2540 for (
unsigned i = 0;
i != Num; ++
i)
2541 Val = (Val <<
W) | (Consts[Num-1-
i]->getZExtValue() &
Mask);
2550 : buildVector32(Elem.
take_front(Num/2), dl, HalfTy, DAG);
2553 : buildVector32(Elem.
drop_front(Num/2), dl, HalfTy, DAG);
2561 MVT VecTy = ty(VecV);
2567 assert((VecWidth % ElemWidth) == 0);
2568 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV);
2572 if (ElemWidth == 1) {
2574 "Vector elements should equal vector width size");
2575 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2586 if (ValWidth == 1) {
2597 unsigned Scale = VecWidth / ValWidth;
2602 unsigned VecRep = 8 / VecWidth;
2611 T1 = expandPredicate(
T1, dl, DAG);
2618 assert(VecWidth == 32 || VecWidth == 64);
2621 MVT ScalarTy = tyScalar(VecTy);
2628 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2629 if (VecWidth == 64 && ValWidth == 32) {
2630 assert(Off == 0 || Off == 32);
2631 unsigned SubIdx =
Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi;
2633 }
else if (Off == 0 && (ValWidth % 8) == 0) {
2640 {VecV, WidthV, OffV});
2648 {VecV, WidthV, OffV});
2661 MVT VecTy = ty(VecV);
2663 MVT ValTy = ty(ValV);
2670 for (
unsigned R = Scale;
R > 1;
R /= 2) {
2671 ValR = contractPredicate(ValR, dl, DAG);
2679 unsigned ValBytes = 64 / Scale;
2685 {VecR, ValR,
Width, Idx});
2691 assert(VecWidth == 32 || VecWidth == 64);
2692 assert((VecWidth % ValWidth) == 0);
2708 unsigned W =
C->getZExtValue() * ValWidth;
2711 {VecV, ValV, WidthV, OffV});
2717 {VecV, ValV, WidthV, OffV});
2724 HexagonTargetLowering::expandPredicate(
SDValue Vec32,
const SDLoc &dl,
2726 assert(ty(Vec32).getSizeInBits() == 32);
2729 return getInstr(Hexagon::S2_vsxtbh, dl,
MVT::i64, {Vec32}, DAG);
2733 HexagonTargetLowering::contractPredicate(
SDValue Vec64,
const SDLoc &dl,
2735 assert(ty(Vec64).getSizeInBits() == 64);
2738 return getInstr(Hexagon::S2_vtrunehb, dl,
MVT::i32, {Vec64}, DAG);
2761 MVT ValTy = ty(Val);
2766 if (ValLen == ResLen)
2769 const SDLoc &dl(Val);
2771 assert(ResLen % ValLen == 0);
2774 for (
unsigned i = 1,
e = ResLen / ValLen;
i <
e; ++
i)
2775 Concats.push_back(DAG.
getUNDEF(ValTy));
2786 for (
unsigned i = 0,
e =
Op.getNumOperands();
i !=
e; ++
i)
2787 Ops.push_back(
Op.getOperand(
i));
2790 return buildVector32(Ops, dl, VecTy, DAG);
2792 return buildVector64(Ops, dl, VecTy, DAG);
2796 bool All0 =
true, All1 =
true;
2798 auto *CN = dyn_cast<ConstantSDNode>(
P.getNode());
2799 if (CN ==
nullptr) {
2800 All0 = All1 =
false;
2819 for (
unsigned i = 0;
i != 8; ++
i) {
2824 for (
unsigned i = 0,
e = A.size()/2;
i !=
e; ++
i)
2828 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2848 MVT OpTy = ty(
Op.getOperand(0));
2852 assert(Scale ==
Op.getNumOperands() && Scale > 1);
2861 for (
SDValue P :
Op.getNode()->op_values()) {
2863 for (
unsigned R = Scale; R > 1; R /= 2) {
2864 W = contractPredicate(
W, dl, DAG);
2869 Words[IdxW].push_back(
W);
2874 Words[IdxW ^ 1].
clear();
2876 for (
unsigned i = 0,
e = Words[IdxW].
size();
i !=
e;
i += 2) {
2877 SDValue W0 = Words[IdxW][
i], W1 = Words[IdxW][
i+1];
2880 {W0, W1, WidthV, WidthV});
2881 Words[IdxW ^ 1].push_back(
T);
2888 assert(Scale == 2 && Words[IdxW].
size() == 2);
2891 Words[IdxW][1], Words[IdxW][0]);
2903 return extractVector(Vec,
Op.getOperand(1),
SDLoc(
Op), ElemTy, ty(
Op), DAG);
2909 return extractVector(
Op.getOperand(0),
Op.getOperand(1),
SDLoc(
Op),
2910 ty(
Op), ty(
Op), DAG);
2916 return insertVector(
Op.getOperand(0),
Op.getOperand(1),
Op.getOperand(2),
2924 return insertVector(
Op.getOperand(0), ValV,
Op.getOperand(2),
2955 LN = cast<LoadSDNode>(
NL.getNode());
2959 if (!validateConstPtrAlignment(LN->
getBasePtr(), ClaimAlign, dl, DAG))
2960 return replaceMemWithUndef(
Op, DAG);
2989 SN = cast<StoreSDNode>(NS.
getNode());
2993 if (!validateConstPtrAlignment(SN->
getBasePtr(), ClaimAlign, dl, DAG))
2994 return replaceMemWithUndef(
Op, DAG);
2998 if (ClaimAlign < NeedAlign)
3007 MVT LoadTy = ty(
Op);
3010 if (HaveAlign >= NeedAlign)
3019 bool DoDefault =
false;
3030 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3049 unsigned LoadLen = NeedAlign;
3052 auto BO = getBaseAndOffset(
Base);
3053 unsigned BaseOpc = BO.first.getOpcode();
3057 if (BO.second % LoadLen != 0) {
3060 BO.second -= BO.second % LoadLen;
3075 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen,
Align(LoadLen),
3076 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
3077 MMO->getSuccessOrdering(), MMO->getFailureOrdering());
3094 auto *CY = dyn_cast<ConstantSDNode>(
Y);
3102 unsigned Opc =
Op.getOpcode();
3106 assert(VY != 0 &&
"This should have been folded");
3131 unsigned Opc =
Op.getOpcode();
3138 EVT CarryTy =
C.getValueType();
3140 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3159 unsigned OffsetReg = Hexagon::R28;
3165 Chain = DAG.
getCopyToReg(Chain, dl, OffsetReg, Offset);
3175 unsigned Opc =
Op.getOpcode();
3181 if (isHvxOperation(
Op.getNode(), DAG)) {
3183 if (
SDValue V = LowerHvxOperation(
Op, DAG))
3190 Op.getNode()->dumpr(&DAG);
3192 errs() <<
"Error: check for a non-legal type in this operation\n";
3242 if (isHvxOperation(
N, DAG)) {
3243 LowerHvxOperationWrapper(
N,
Results, DAG);
3261 if (isHvxOperation(
N, DAG)) {
3262 ReplaceHvxNodeResults(
N,
Results, DAG);
3268 switch (
N->getOpcode()) {
3275 if (
N->getValueType(0) ==
MVT::i8) {
3276 if (
N->getOperand(0).getValueType() ==
MVT::v8i1) {
3278 N->getOperand(0), DAG);
3290 if (isHvxOperation(
N, DCI.
DAG)) {
3291 if (
SDValue V = PerformHvxDAGCombine(
N, DCI))
3301 unsigned Opc =
Op.getOpcode();
3305 switch (
P.getOpcode()) {
3309 return getZero(dl, ty(
Op), DCI.
DAG);
3322 Op.getOperand(2),
Op.getOperand(1));
3335 int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3347 if (Constraint.
size() == 1) {
3348 switch (Constraint[0]) {
3363 std::pair<unsigned, const TargetRegisterClass*>
3367 if (Constraint.
size() == 1) {
3368 switch (Constraint[0]) {
3372 return {0u,
nullptr};
3378 return {0u, &Hexagon::IntRegsRegClass};
3381 return {0u, &Hexagon::DoubleRegsRegClass};
3386 return {0u,
nullptr};
3387 return {0u, &Hexagon::ModRegsRegClass};
3391 return {0u,
nullptr};
3394 return {0u, &Hexagon::HvxQRRegClass};
3400 return {0u,
nullptr};
3402 return {0u, &Hexagon::HvxVRRegClass};
3405 return {0u, &Hexagon::HvxVRRegClass};
3406 return {0u, &Hexagon::HvxWRRegClass};
3408 return {0u, &Hexagon::HvxWRRegClass};
3412 return {0u,
nullptr};
3423 bool ForCodeSize)
const {
3440 Align A =
DL.getABITypeAlign(Ty);
3453 int Scale = AM.
Scale;
3477 return Imm >= -512 && Imm <= 511;
3487 bool IsCalleeStructRet,
3488 bool IsCallerStructRet,
3495 bool CCMatch = CallerCC == CalleeCC;
3503 if (!isa<GlobalAddressSDNode>(
Callee) &&
3504 !isa<ExternalSymbolSDNode>(
Callee)) {
3524 if (IsCalleeStructRet || IsCallerStructRet)
3547 if (
Op.size() >= 8 &&
Op.isAligned(
Align(8)))
3549 if (
Op.size() >= 4 &&
Op.isAligned(
Align(4)))
3551 if (
Op.size() >= 2 &&
Op.isAligned(
Align(2)))
3561 return allowsHvxMemoryAccess(SVT, Flags, Fast);
3563 Context,
DL, VT, AddrSpace, Alignment, Flags, Fast);
3571 return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3577 std::pair<const TargetRegisterClass*, uint8_t>
3585 return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3587 return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3589 return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3601 auto *L = cast<LoadSDNode>(
Load);
3602 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3608 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3618 Module *
M =
BB->getParent()->getParent();
3620 assert((SZ == 32 || SZ == 64) &&
"Only 32/64-bit atomic loads supported");
3621 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3622 : Intrinsic::hexagon_L4_loadd_locked;
3625 auto PtrTy = cast<PointerType>(
Addr->getType());
3627 Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace());
3632 return Builder.CreateBitCast(Call, ValueTy);
3641 Module *
M =
BB->getParent()->getParent();
3646 assert((SZ == 32 || SZ == 64) &&
"Only 32/64-bit atomic stores supported");
3647 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3648 : Intrinsic::hexagon_S4_stored_locked;
3651 unsigned AS =
Addr->getType()->getPointerAddressSpace();
3653 Val =
Builder.CreateBitCast(Val, CastTy);
3672 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64