44 #include "llvm/IR/IntrinsicsHexagon.h"
66 #define DEBUG_TYPE "hexagon-lowering"
70 cl::desc(
"Control jump table emission on Hexagon target"));
74 cl::desc(
"Enable Hexagon SDNode scheduling"));
78 cl::desc(
"Enable Fast Math processing"));
82 cl::desc(
"Set minimum jump tables"));
86 cl::desc(
"Max #stores to inline memcpy"));
90 cl::desc(
"Max #stores to inline memcpy"));
94 cl::desc(
"Max #stores to inline memmove"));
98 cl::desc(
"Max #stores to inline memmove"));
102 cl::desc(
"Max #stores to inline memset"));
106 cl::desc(
"Max #stores to inline memset"));
110 cl::desc(
"Rewrite unaligned loads as a pair of aligned loads"));
115 cl::desc(
"Disable minimum alignment of 1 for "
116 "arguments passed by value on stack"));
120 class HexagonCCState :
public CCState {
121 unsigned NumNamedVarArgParams = 0;
126 unsigned NumNamedArgs)
127 :
CCState(CC, IsVarArg, MF, locs,
C),
128 NumNamedVarArgParams(NumNamedArgs) {}
129 unsigned getNumNamedVarArgParams()
const {
return NumNamedVarArgParams; }
148 if (RegNum != NumArgRegs && RegNum % 2 == 1)
157 #include "HexagonGenCallingConv.inc"
190 return CCInfo.
CheckReturn(Outs, RetCC_Hexagon_HVX);
220 for (
unsigned i = 0;
i != RVLocs.size(); ++
i) {
234 RetOps.push_back(
Flag);
248 .
Case(
"r0", Hexagon::R0)
249 .
Case(
"r1", Hexagon::R1)
251 .
Case(
"r3", Hexagon::R3)
253 .
Case(
"r5", Hexagon::R5)
255 .
Case(
"r7", Hexagon::R7)
256 .
Case(
"r8", Hexagon::R8)
257 .
Case(
"r9", Hexagon::R9)
258 .
Case(
"r10", Hexagon::R10)
259 .
Case(
"r11", Hexagon::R11)
260 .
Case(
"r12", Hexagon::R12)
261 .
Case(
"r13", Hexagon::R13)
262 .
Case(
"r14", Hexagon::R14)
263 .
Case(
"r15", Hexagon::R15)
264 .
Case(
"r16", Hexagon::R16)
265 .
Case(
"r17", Hexagon::R17)
266 .
Case(
"r18", Hexagon::R18)
267 .
Case(
"r19", Hexagon::R19)
268 .
Case(
"r20", Hexagon::R20)
269 .
Case(
"r21", Hexagon::R21)
270 .
Case(
"r22", Hexagon::R22)
271 .
Case(
"r23", Hexagon::R23)
272 .
Case(
"r24", Hexagon::R24)
273 .
Case(
"r25", Hexagon::R25)
274 .
Case(
"r26", Hexagon::R26)
275 .
Case(
"r27", Hexagon::R27)
276 .
Case(
"r28", Hexagon::R28)
277 .
Case(
"r29", Hexagon::R29)
278 .
Case(
"r30", Hexagon::R30)
279 .
Case(
"r31", Hexagon::R31)
280 .
Case(
"r1:0", Hexagon::D0)
281 .
Case(
"r3:2", Hexagon::D1)
282 .
Case(
"r5:4", Hexagon::D2)
283 .
Case(
"r7:6", Hexagon::D3)
284 .
Case(
"r9:8", Hexagon::D4)
285 .
Case(
"r11:10", Hexagon::D5)
286 .
Case(
"r13:12", Hexagon::D6)
287 .
Case(
"r15:14", Hexagon::D7)
288 .
Case(
"r17:16", Hexagon::D8)
289 .
Case(
"r19:18", Hexagon::D9)
290 .
Case(
"r21:20", Hexagon::D10)
291 .
Case(
"r23:22", Hexagon::D11)
292 .
Case(
"r25:24", Hexagon::D12)
293 .
Case(
"r27:26", Hexagon::D13)
294 .
Case(
"r29:28", Hexagon::D14)
295 .
Case(
"r31:30", Hexagon::D15)
296 .
Case(
"sp", Hexagon::R29)
297 .
Case(
"fp", Hexagon::R30)
298 .
Case(
"lr", Hexagon::R31)
299 .
Case(
"p0", Hexagon::P0)
300 .
Case(
"p1", Hexagon::P1)
302 .
Case(
"p3", Hexagon::P3)
303 .
Case(
"sa0", Hexagon::SA0)
305 .
Case(
"sa1", Hexagon::SA1)
306 .
Case(
"lc1", Hexagon::LC1)
309 .
Case(
"usr", Hexagon::USR)
310 .
Case(
"ugp", Hexagon::UGP)
311 .
Case(
"cs0", Hexagon::CS0)
312 .
Case(
"cs1", Hexagon::CS1)
342 for (
unsigned i = 0;
i != RVLocs.size(); ++
i) {
344 if (RVLocs[
i].getValVT() ==
MVT::i1) {
366 RVLocs[
i].getValVT(), Glue);
370 InVals.push_back(RetVal.
getValue(0));
392 bool IsStructRet = Outs.empty() ?
false : Outs[0].Flags.isSRet();
406 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.
getContext(),
410 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
412 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
414 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
419 IsVarArg, IsStructRet, StructAttrFlag, Outs,
421 for (
unsigned i = 0,
e = ArgLocs.size();
i !=
e; ++
i) {
429 :
"Argument must be passed on stack. "
430 "Not eligible for Tail Call\n"));
433 unsigned NumBytes = CCInfo.getNextStackOffset();
441 bool NeedsArgAlign =
false;
442 Align LargestAlignSeen;
444 for (
unsigned i = 0,
e = ArgLocs.size();
i !=
e; ++
i) {
450 NeedsArgAlign |= ArgAlign;
476 StackPtr.getValueType());
490 MemOpChains.push_back(
S);
498 RegsToPass.push_back(std::make_pair(VA.
getLocReg(),
Arg));
501 if (NeedsArgAlign && Subtarget.
hasV60Ops()) {
502 LLVM_DEBUG(
dbgs() <<
"Function needs byte stack align due to call args\n");
503 Align VecAlign(HRI.getSpillAlignment(Hexagon::HvxVRRegClass));
504 LargestAlignSeen =
std::max(LargestAlignSeen, VecAlign);
509 if (!MemOpChains.empty())
523 for (
unsigned i = 0,
e = RegsToPass.size();
i !=
e; ++
i) {
525 RegsToPass[
i].second, Glue);
540 for (
unsigned i = 0,
e = RegsToPass.size();
i !=
e; ++
i) {
542 RegsToPass[
i].second, Glue);
557 dyn_cast<ExternalSymbolSDNode>(
Callee)) {
564 Ops.push_back(Chain);
569 for (
unsigned i = 0,
e = RegsToPass.size();
i !=
e; ++
i) {
571 RegsToPass[
i].second.getValueType()));
575 assert(
Mask &&
"Missing call preserved mask for calling convention");
592 Chain = DAG.
getNode(OpCode, dl, NodeTys, Ops);
628 Base =
Op->getOperand(0);
630 if (!isa<ConstantSDNode>(
Offset.getNode()))
634 int32_t V = cast<ConstantSDNode>(
Offset.getNode())->getSExtValue();
649 unsigned NumOps =
Op.getNumOperands();
650 if (
Op.getOperand(NumOps-1).getValueType() ==
MVT::Glue)
654 unsigned Flags = cast<ConstantSDNode>(
Op.getOperand(
i))->getZExtValue();
669 for (; NumVals; --NumVals, ++
i) {
670 unsigned Reg = cast<RegisterSDNode>(
Op.getOperand(
i))->getReg();
673 HMFI.setHasClobberLR(
true);
713 unsigned IntNo = cast<ConstantSDNode>(
Op.getOperand(1))->getZExtValue();
715 if (IntNo == Intrinsic::hexagon_prefetch) {
733 assert(AlignConst &&
"Non-constant Align in LowerDYNAMIC_STACKALLOC");
739 A = HFI.getStackAlign().value();
742 dbgs () << __func__ <<
" Align: " << A <<
" Size: ";
743 Size.getNode()->dump(&DAG);
768 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
773 CCInfo.AnalyzeFormalArguments(
Ins, CC_Hexagon_HVX);
775 CCInfo.AnalyzeFormalArguments(
Ins, CC_Hexagon_Legacy);
777 CCInfo.AnalyzeFormalArguments(
Ins, CC_Hexagon);
786 switch (RC.
getID()) {
787 case Hexagon::IntRegsRegClassID:
788 return Reg - Hexagon::R0 + 1;
789 case Hexagon::DoubleRegsRegClassID:
790 return (
Reg - Hexagon::D0 + 1) * 2;
791 case Hexagon::HvxVRRegClassID:
792 return Reg - Hexagon::V0 + 1;
793 case Hexagon::HvxWRRegClassID:
794 return (
Reg - Hexagon::W0 + 1) * 2;
801 HFL.FirstVarArgSavedReg = 0;
804 for (
unsigned i = 0,
e = ArgLocs.size();
i !=
e; ++
i) {
840 assert(RegSize == 32 || RegSize == 64 ||
844 InVals.push_back(Copy);
846 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.
getLocReg());
852 unsigned ObjSize = Flags.
isByVal()
865 InVals.push_back(FIN);
875 for (
int i = HFL.FirstVarArgSavedReg;
i < 6;
i++)
880 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
884 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
885 bool RequiresPadding = (NumVarArgRegs & 1);
886 int RegSaveAreaSizePlusPadding = RequiresPadding
887 ? (NumVarArgRegs + 1) * 4
890 if (RegSaveAreaSizePlusPadding > 0) {
893 if (!(RegAreaStart % 8))
894 RegAreaStart = (RegAreaStart + 7) & -8;
896 int RegSaveAreaFrameIndex =
898 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
901 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
903 HMFI.setVarArgsFrameIndex(FI);
909 HMFI.setRegSavedAreaStartFrameIndex(FI);
910 HMFI.setVarArgsFrameIndex(FI);
919 HMFI.setVarArgsFrameIndex(FI);
932 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
952 SDValue SavedRegAreaStartFrameIndex =
957 if (HFL.FirstVarArgSavedReg & 1)
958 SavedRegAreaStartFrameIndex =
967 SavedRegAreaStartFrameIndex,
969 MemOps.push_back(
Store);
978 MemOps.push_back(
Store);
987 MemOps.push_back(
Store);
999 const Value *DestSV = cast<SrcValueSDNode>(
Op.getOperand(3))->getValue();
1000 const Value *SrcSV = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
1006 false,
false,
false,
1035 auto isSExtFree = [
this](
SDValue N) {
1036 switch (
N.getOpcode()) {
1042 EVT OrigTy = cast<VTSDNode>(
Op.getOperand(1))->getVT();
1048 return ThisBW >= OrigBW;
1059 bool IsNegative =
C &&
C->getAPIntValue().isNegative();
1060 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1072 SDValue Op1 =
Op.getOperand(1), Op2 =
Op.getOperand(2);
1094 EVT ValTy =
Op.getValueType();
1097 bool isVTi1Type =
false;
1098 if (
auto *CV = dyn_cast<ConstantVector>(CPN->
getConstVal())) {
1099 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1102 unsigned VecLen = CV->getNumOperands();
1104 "conversion only supported for pow2 VectorSize");
1105 for (
unsigned i = 0;
i < VecLen; ++
i)
1106 NewConst.push_back(IRB.
getInt8(CV->getOperand(
i)->isZeroValue()));
1121 else if (isVTi1Type)
1127 assert(cast<ConstantPoolSDNode>(
T)->getTargetFlags() == TF &&
1128 "Inconsistent target flag encountered");
1130 if (IsPositionIndependent)
1137 EVT VT =
Op.getValueType();
1138 int Idx = cast<JumpTableSDNode>(
Op)->getIndex();
1158 EVT VT =
Op.getValueType();
1160 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
1180 EVT VT =
Op.getValueType();
1182 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
1200 auto *GAN = cast<GlobalAddressSDNode>(
Op);
1202 auto *GV = GAN->getGlobal();
1203 int64_t
Offset = GAN->getOffset();
1211 if (GO && Subtarget.
useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1233 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
1278 assert(
Mask &&
"Missing call preserved mask for calling convention");
1313 if (IsPositionIndependent) {
1386 Hexagon::R0, Flags);
1522 for (
unsigned LegalIntOp :
1562 for (
unsigned IntExpOp :
1571 for (
unsigned FPExpOp :
1607 static const unsigned VectExpOps[] = {
1633 for (
unsigned VectExpOp : VectExpOps)
1647 if (VT.getVectorElementType() !=
MVT::i32) {
1687 if (NativeVT.getVectorElementType() !=
MVT::i1)
1779 initializeHVXLowering();
1799 setLibcallName(RTLIB::FPTOUINT_F32_I128,
"__hexagon_fixunssfti");
1800 setLibcallName(RTLIB::FPTOUINT_F64_I128,
"__hexagon_fixunsdfti");
1888 HexagonTargetLowering::validateConstPtrAlignment(
SDValue Ptr,
const SDLoc &dl,
1889 unsigned NeedAlign)
const {
1890 auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1893 unsigned Addr = CA->getZExtValue();
1895 if (HaveAlign < NeedAlign) {
1899 <<
" has alignment " << HaveAlign
1900 <<
", but the memory access requires " << NeedAlign;
1910 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1911 return (
ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1912 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1913 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1914 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1915 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1916 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1925 V = cast<Operator>(V)->getOperand(0);
1927 V = cast<Instruction>(V)->getOperand(0);
1940 if (Blk == Parent) {
1945 BaseVal = BackEdgeVal;
1947 }
while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
1950 if (IntrBaseVal == BackEdgeVal)
1957 assert(Idx >= 0 &&
"Unexpected index to incoming argument in PHI");
1965 Value *IntrBaseVal = V;
1972 }
while (BaseVal != V);
1975 if (
const PHINode *PN = dyn_cast<PHINode>(V))
1989 unsigned Intrinsic)
const {
1990 switch (Intrinsic) {
1991 case Intrinsic::hexagon_L2_loadrd_pbr:
1992 case Intrinsic::hexagon_L2_loadri_pbr:
1993 case Intrinsic::hexagon_L2_loadrh_pbr:
1994 case Intrinsic::hexagon_L2_loadruh_pbr:
1995 case Intrinsic::hexagon_L2_loadrb_pbr:
1996 case Intrinsic::hexagon_L2_loadrub_pbr: {
1998 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
1999 auto &Cont =
I.getCalledFunction()->getParent()->getContext();
2003 Type *ElTy =
I.getCalledFunction()->getReturnType()->getStructElementType(0);
2010 Info.align =
DL.getABITypeAlign(
Info.memVT.getTypeForEVT(Cont));
2014 case Intrinsic::hexagon_V6_vgathermw:
2015 case Intrinsic::hexagon_V6_vgathermw_128B:
2016 case Intrinsic::hexagon_V6_vgathermh:
2017 case Intrinsic::hexagon_V6_vgathermh_128B:
2018 case Intrinsic::hexagon_V6_vgathermhw:
2019 case Intrinsic::hexagon_V6_vgathermhw_128B:
2020 case Intrinsic::hexagon_V6_vgathermwq:
2021 case Intrinsic::hexagon_V6_vgathermwq_128B:
2022 case Intrinsic::hexagon_V6_vgathermhq:
2023 case Intrinsic::hexagon_V6_vgathermhq_128B:
2024 case Intrinsic::hexagon_V6_vgathermhwq:
2025 case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2026 const Module &
M = *
I.getParent()->getParent()->getParent();
2028 Type *VecTy =
I.getArgOperand(1)->getType();
2030 Info.ptrVal =
I.getArgOperand(0);
2033 MaybeAlign(
M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2046 return X.getValueType().isScalarInteger();
2066 unsigned DefinedValues)
const {
2084 unsigned Action = getPreferredHvxVectorAction(VT);
2096 std::pair<SDValue, int>
2097 HexagonTargetLowering::getBaseAndOffset(
SDValue Addr)
const {
2100 if (
auto *CN = dyn_cast<const ConstantSDNode>(Op1.
getNode()))
2101 return {
Addr.getOperand(0), CN->getSExtValue() };
2111 const auto *SVN = cast<ShuffleVectorSDNode>(
Op);
2113 assert(AM.
size() <= 8 &&
"Unexpected shuffle mask");
2114 unsigned VecLen = AM.
size();
2118 "HVX shuffles should be legal");
2128 if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2137 if (AM[
F] >=
int(VecLen)) {
2145 for (
unsigned i = 0,
e =
Mask.size();
i !=
e; ++
i) {
2148 for (
unsigned j = 0;
j != ElemBytes; ++
j)
2149 ByteMask.push_back(-1);
2151 for (
unsigned j = 0;
j != ElemBytes; ++
j)
2152 ByteMask.push_back(
M*ElemBytes +
j);
2155 assert(ByteMask.size() <= 8);
2162 uint64_t MaskIdx = 0;
2163 uint64_t MaskUnd = 0;
2164 for (
unsigned i = 0,
e = ByteMask.size();
i !=
e; ++
i) {
2166 uint64_t
M = ByteMask[
i] & 0xFF;
2172 if (ByteMask.size() == 4) {
2174 if (MaskIdx == (0x03020100 | MaskUnd))
2177 if (MaskIdx == (0x00010203 | MaskUnd)) {
2185 typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0});
2186 if (MaskIdx == (0x06040200 | MaskUnd))
2187 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2188 if (MaskIdx == (0x07050301 | MaskUnd))
2189 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2192 typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1});
2193 if (MaskIdx == (0x02000604 | MaskUnd))
2194 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2195 if (MaskIdx == (0x03010705 | MaskUnd))
2196 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2199 if (ByteMask.size() == 8) {
2201 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2204 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2211 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2212 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2213 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2214 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2215 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2216 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2217 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2218 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2219 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2220 VectorPair
P = opSplit(Op0, dl, DAG);
2221 return getInstr(Hexagon::S2_packhl, dl, VecTy, {
P.second,
P.first}, DAG);
2225 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2226 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2227 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2228 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2239 switch (
Op.getOpcode()) {
2260 return DAG.
getNode(NewOpc, dl, ty(
Op), Op0,
S);
2270 return getVectorShiftByInt(
Op, DAG);
2275 if (isa<ConstantSDNode>(
Op.getOperand(1).getNode()))
2284 MVT InpTy = ty(InpV);
2293 return getInstr(Hexagon::C2_tfrrp, dl, ResTy,
Ext, DAG);
2308 bool AllConst =
true;
2310 for (
unsigned i = 0,
e = Values.
size();
i !=
e; ++
i) {
2317 if (
auto *CN = dyn_cast<ConstantSDNode>(V.
getNode())) {
2318 const ConstantInt *CI = CN->getConstantIntValue();
2320 }
else if (
auto *CN = dyn_cast<ConstantFPSDNode>(V.
getNode())) {
2321 const ConstantFP *CF = CN->getConstantFPValue();
2338 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2342 if (!isUndef(Elem[First]))
2350 return getZero(dl, VecTy, DAG);
2355 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2356 Consts[1]->getZExtValue() << 16;
2360 {Elem[1], Elem[0]}, DAG);
2367 int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2368 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2369 (Consts[1]->getZExtValue() & 0xFF) << 16 |
2370 Consts[2]->getZExtValue() << 24;
2375 bool IsSplat =
true;
2376 for (
unsigned i = First+1;
i != Num; ++
i) {
2377 if (Elem[
i] == Elem[First] || isUndef(Elem[
i]))
2393 for (
unsigned i = 0;
i != 4; ++
i) {
2420 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2424 if (!isUndef(Elem[First]))
2432 return getZero(dl, VecTy, DAG);
2436 bool IsSplat =
true;
2437 for (
unsigned i = First+1;
i != Num; ++
i) {
2438 if (Elem[
i] == Elem[First] || isUndef(Elem[
i]))
2455 : (ElemTy ==
MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull;
2456 for (
unsigned i = 0;
i != Num; ++
i)
2457 Val = (Val <<
W) | (Consts[Num-1-
i]->getZExtValue() &
Mask);
2466 : buildVector32(Elem.
take_front(Num/2), dl, HalfTy, DAG);
2469 : buildVector32(Elem.
drop_front(Num/2), dl, HalfTy, DAG);
2477 MVT VecTy = ty(VecV);
2483 assert((VecWidth % ElemWidth) == 0);
2484 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV);
2488 if (ElemWidth == 1) {
2490 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2501 if (ValWidth == 1) {
2512 unsigned Scale = VecWidth / ValWidth;
2517 unsigned VecRep = 8 / VecWidth;
2526 T1 = expandPredicate(
T1, dl, DAG);
2533 assert(VecWidth == 32 || VecWidth == 64);
2536 MVT ScalarTy = tyScalar(VecTy);
2543 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2544 if (VecWidth == 64 && ValWidth == 32) {
2545 assert(Off == 0 || Off == 32);
2546 unsigned SubIdx =
Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi;
2548 }
else if (Off == 0 && (ValWidth % 8) == 0) {
2555 {VecV, WidthV, OffV});
2563 {VecV, WidthV, OffV});
2576 MVT VecTy = ty(VecV);
2578 MVT ValTy = ty(ValV);
2585 for (
unsigned R = Scale;
R > 1;
R /= 2) {
2586 ValR = contractPredicate(ValR, dl, DAG);
2594 unsigned ValBytes = 64 / Scale;
2600 {VecR, ValR,
Width, Idx});
2606 assert(VecWidth == 32 || VecWidth == 64);
2607 assert((VecWidth % ValWidth) == 0);
2623 unsigned W =
C->getZExtValue() * ValWidth;
2626 {VecV, ValV, WidthV, OffV});
2632 {VecV, ValV, WidthV, OffV});
2639 HexagonTargetLowering::expandPredicate(
SDValue Vec32,
const SDLoc &dl,
2641 assert(ty(Vec32).getSizeInBits() == 32);
2644 return getInstr(Hexagon::S2_vsxtbh, dl,
MVT::i64, {Vec32}, DAG);
2648 HexagonTargetLowering::contractPredicate(
SDValue Vec64,
const SDLoc &dl,
2650 assert(ty(Vec64).getSizeInBits() == 64);
2653 return getInstr(Hexagon::S2_vtrunehb, dl,
MVT::i32, {Vec64}, DAG);
2677 MVT ValTy = ty(Val);
2682 if (ValLen == ResLen)
2685 const SDLoc &dl(Val);
2687 assert(ResLen % ValLen == 0);
2690 for (
unsigned i = 1,
e = ResLen / ValLen;
i <
e; ++
i)
2691 Concats.push_back(DAG.
getUNDEF(ValTy));
2702 for (
unsigned i = 0,
e =
Op.getNumOperands();
i !=
e; ++
i)
2703 Ops.push_back(
Op.getOperand(
i));
2706 return buildVector32(Ops, dl, VecTy, DAG);
2708 return buildVector64(Ops, dl, VecTy, DAG);
2712 bool All0 =
true, All1 =
true;
2714 auto *CN = dyn_cast<ConstantSDNode>(
P.getNode());
2715 if (CN ==
nullptr) {
2716 All0 = All1 =
false;
2735 for (
unsigned i = 0;
i != 8; ++
i) {
2740 for (
unsigned i = 0,
e = A.size()/2;
i !=
e; ++
i)
2744 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2764 MVT OpTy = ty(
Op.getOperand(0));
2768 assert(Scale ==
Op.getNumOperands() && Scale > 1);
2777 for (
SDValue P :
Op.getNode()->op_values()) {
2779 for (
unsigned R = Scale; R > 1; R /= 2) {
2780 W = contractPredicate(
W, dl, DAG);
2785 Words[IdxW].push_back(
W);
2790 Words[IdxW ^ 1].
clear();
2792 for (
unsigned i = 0,
e = Words[IdxW].
size();
i !=
e;
i += 2) {
2793 SDValue W0 = Words[IdxW][
i], W1 = Words[IdxW][
i+1];
2796 {W0, W1, WidthV, WidthV});
2797 Words[IdxW ^ 1].push_back(
T);
2805 assert(Scale == 2 && Words[IdxW].
size() == 2);
2808 Words[IdxW][1], Words[IdxW][0]);
2820 return extractVector(Vec,
Op.getOperand(1),
SDLoc(
Op), ElemTy, ty(
Op), DAG);
2826 return extractVector(
Op.getOperand(0),
Op.getOperand(1),
SDLoc(
Op),
2827 ty(
Op), ty(
Op), DAG);
2833 return insertVector(
Op.getOperand(0),
Op.getOperand(1),
Op.getOperand(2),
2834 SDLoc(
Op), ty(
Op).getVectorElementType(), DAG);
2841 return insertVector(
Op.getOperand(0), ValV,
Op.getOperand(2),
2874 validateConstPtrAlignment(Ptr, dl, ClaimAlign);
2878 if (ClaimAlign < NeedAlign)
2887 MVT LoadTy = ty(
Op);
2890 if (HaveAlign >= NeedAlign)
2899 bool DoDefault =
false;
2910 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
2929 unsigned LoadLen = NeedAlign;
2932 auto BO = getBaseAndOffset(Base);
2933 unsigned BaseOpc = BO.first.getOpcode();
2937 if (BO.second % LoadLen != 0) {
2940 BO.second -= BO.second % LoadLen;
2955 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen,
Align(LoadLen),
2956 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
2957 MMO->getOrdering(), MMO->getFailureOrdering());
2974 auto *CY = dyn_cast<ConstantSDNode>(
Y);
2982 unsigned Opc =
Op.getOpcode();
2986 assert(VY != 0 &&
"This should have been folded");
3011 unsigned Opc =
Op.getOpcode();
3018 EVT CarryTy =
C.getValueType();
3020 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3039 unsigned OffsetReg = Hexagon::R28;
3055 unsigned Opc =
Op.getOpcode();
3061 if (isHvxOperation(
Op.getNode(), DAG)) {
3063 if (
SDValue V = LowerHvxOperation(
Op, DAG))
3070 Op.getNode()->dumpr(&DAG);
3072 errs() <<
"Error: check for a non-legal type in this operation\n";
3122 if (isHvxOperation(
N, DAG)) {
3123 LowerHvxOperationWrapper(
N,
Results, DAG);
3141 if (isHvxOperation(
N, DAG)) {
3142 ReplaceHvxNodeResults(
N,
Results, DAG);
3148 switch (
N->getOpcode()) {
3155 if (
N->getValueType(0) ==
MVT::i8) {
3156 if (
N->getOperand(0).getValueType() ==
MVT::v8i1) {
3158 N->getOperand(0), DAG);
3170 if (isHvxOperation(
N, DCI.
DAG)) {
3171 if (
SDValue V = PerformHvxDAGCombine(
N, DCI))
3181 unsigned Opc =
Op.getOpcode();
3185 switch (
P.getOpcode()) {
3189 return getZero(dl, ty(
Op), DCI.
DAG);
3202 Op.getOperand(2),
Op.getOperand(1));
3215 int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3227 if (Constraint.
size() == 1) {
3228 switch (Constraint[0]) {
3243 std::pair<unsigned, const TargetRegisterClass*>
3247 if (Constraint.
size() == 1) {
3248 switch (Constraint[0]) {
3252 return {0u,
nullptr};
3258 return {0u, &Hexagon::IntRegsRegClass};
3261 return {0u, &Hexagon::DoubleRegsRegClass};
3266 return {0u,
nullptr};
3267 return {0u, &Hexagon::ModRegsRegClass};
3271 return {0u,
nullptr};
3274 return {0u, &Hexagon::HvxQRRegClass};
3280 return {0u,
nullptr};
3282 return {0u, &Hexagon::HvxVRRegClass};
3285 return {0u, &Hexagon::HvxVRRegClass};
3286 return {0u, &Hexagon::HvxWRRegClass};
3288 return {0u, &Hexagon::HvxWRRegClass};
3292 return {0u,
nullptr};
3303 bool ForCodeSize)
const {
3320 Align A =
DL.getABITypeAlign(Ty);
3333 int Scale = AM.
Scale;
3357 return Imm >= -512 && Imm <= 511;
3367 bool IsCalleeStructRet,
3368 bool IsCallerStructRet,
3375 bool CCMatch = CallerCC == CalleeCC;
3383 if (!isa<GlobalAddressSDNode>(
Callee) &&
3384 !isa<ExternalSymbolSDNode>(
Callee)) {
3404 if (IsCalleeStructRet || IsCallerStructRet)
3427 if (
Op.size() >= 8 &&
Op.isAligned(
Align(8)))
3429 if (
Op.size() >= 4 &&
Op.isAligned(
Align(4)))
3431 if (
Op.size() >= 2 &&
Op.isAligned(
Align(2)))
3441 return allowsHvxMemoryAccess(SVT, Flags, Fast);
3443 Context,
DL, VT, AddrSpace, Alignment, Flags, Fast);
3451 return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3457 std::pair<const TargetRegisterClass*, uint8_t>
3465 return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3467 return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3469 return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3481 auto *L = cast<LoadSDNode>(
Load);
3482 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3488 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3497 Module *
M =
BB->getParent()->getParent();
3498 auto PT = cast<PointerType>(
Addr->getType());
3499 Type *Ty = PT->getElementType();
3501 assert((SZ == 32 || SZ == 64) &&
"Only 32/64-bit atomic loads supported");
3502 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3503 : Intrinsic::hexagon_L4_loadd_locked;
3507 =
Builder.getIntNTy(SZ)->getPointerTo(PT->getAddressSpace());
3512 return Builder.CreateBitCast(Call, Ty);
3520 Module *
M =
BB->getParent()->getParent();
3525 assert((SZ == 32 || SZ == 64) &&
"Only 32/64-bit atomic stores supported");
3526 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3527 : Intrinsic::hexagon_S4_stored_locked;
3530 unsigned AS =
Addr->getType()->getPointerAddressSpace();
3532 Val =
Builder.CreateBitCast(Val, CastTy);
3550 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64;