23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
28struct SystemZAddressingMode {
63 bool IncludesDynAlloc;
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 :
Form(form), DR(dr), Disp(0), IncludesDynAlloc(
false) {}
69 bool hasIndexField() {
return Form != FormBD; }
72 bool isDynAlloc() {
return Form == FormBDXDynAlloc; }
75 errs() <<
"SystemZAddressingMode " <<
this <<
'\n';
79 Base.getNode()->dump(DAG);
83 if (hasIndexField()) {
86 Index.getNode()->dump(DAG);
91 errs() <<
" Disp " << Disp;
93 errs() <<
" + ADJDYNALLOC";
119struct RxSBGOperands {
121 : Opcode(
Op), BitSize(
N.getValueSizeInBits()),
122 Mask(
allOnes(BitSize)), Input(
N), Start(64 - BitSize),
End(63),
139 return CurDAG->getTargetConstant(Imm,
SDLoc(
Node),
Node->getValueType(0));
152 bool expandAddress(SystemZAddressingMode &AM,
bool IsBase)
const;
155 bool selectAddress(
SDValue N, SystemZAddressingMode &AM)
const;
158 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
160 void getAddressOperands(
const SystemZAddressingMode &AM,
EVT VT,
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR,
SDValue Addr,
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
193 return selectBDAddr(SystemZAddressingMode::Disp12Only,
Addr,
Base, Disp);
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
199 return selectBDAddr(SystemZAddressingMode::Disp20Only,
Addr,
Base, Disp);
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair,
Addr,
Base, Disp);
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair,
Addr,
Base, Disp);
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
277 bool refineRxSBGMask(RxSBGOperands &RxSBG,
uint64_t Mask)
const;
281 bool expandRxSBG(RxSBGOperands &RxSBG)
const;
295 bool tryRxSBG(
SDNode *
N,
unsigned Opcode);
313 bool tryGather(
SDNode *
N,
unsigned Opcode);
316 bool tryScatter(
StoreSDNode *Store,
unsigned Opcode);
321 bool tryFoldLoadStoreIntoMemOperand(
SDNode *
Node);
336 bool storeLoadCanUseMVC(
SDNode *
N)
const;
341 bool storeLoadCanUseBlockBinary(
SDNode *
N,
unsigned I)
const;
345 bool storeLoadIsAligned(
SDNode *
N)
const;
353 SystemZDAGToDAGISel() =
delete;
360 if (
F.getFnAttribute(
"fentry-call").getValueAsString() !=
"true") {
361 if (
F.hasFnAttribute(
"mnop-mcount"))
363 if (
F.hasFnAttribute(
"mrecord-mcount"))
375 std::vector<SDValue> &OutOps)
override;
380 #include "SystemZGenDAGISel.inc"
384char SystemZDAGToDAGISel::ID = 0;
390 return new SystemZDAGToDAGISel(
TM, OptLevel);
396static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
398 case SystemZAddressingMode::Disp12Only:
399 return isUInt<12>(Val);
401 case SystemZAddressingMode::Disp12Pair:
402 case SystemZAddressingMode::Disp20Only:
403 case SystemZAddressingMode::Disp20Pair:
404 return isInt<20>(Val);
406 case SystemZAddressingMode::Disp20Only128:
407 return isInt<20>(Val) && isInt<20>(Val + 8);
427 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
429 AM.IncludesDynAlloc =
true;
439 if (AM.hasIndexField() && !AM.Index.getNode()) {
449static bool expandDisp(SystemZAddressingMode &AM,
bool IsBase,
452 int64_t TestDisp = AM.Disp + Op1;
464bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
467 unsigned Opcode =
N.getOpcode();
469 if (Opcode ==
ISD::TRUNCATE &&
N.getOperand(0).getValueSizeInBits() <= 64) {
471 Opcode =
N.getOpcode();
473 if (Opcode ==
ISD::ADD || CurDAG->isBaseWithConstantOffset(
N)) {
487 cast<ConstantSDNode>(Op0)->getSExtValue());
490 cast<ConstantSDNode>(Op1)->getSExtValue());
500 cast<GlobalAddressSDNode>(Anchor)->getOffset());
508static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
511 case SystemZAddressingMode::Disp12Only:
512 case SystemZAddressingMode::Disp20Only:
513 case SystemZAddressingMode::Disp20Only128:
516 case SystemZAddressingMode::Disp12Pair:
518 return isUInt<12>(Val);
520 case SystemZAddressingMode::Disp20Pair:
522 return !isUInt<12>(Val);
546 if (isUInt<12>(Disp))
551 if (!isInt<16>(Disp))
560 if (
Index->hasOneUse())
565 unsigned IndexOpcode =
Index->getOpcode();
573 if (
Base->hasOneUse())
580bool SystemZDAGToDAGISel::selectAddress(
SDValue Addr,
581 SystemZAddressingMode &AM)
const {
589 cast<ConstantSDNode>(
Addr)->getSExtValue()))
597 while (expandAddress(AM,
true) ||
598 (AM.Index.getNode() && expandAddress(AM,
false)))
602 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
603 !
shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
611 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
624 if (
N->getNodeId() == -1 ||
637void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
643 Base = CurDAG->getRegister(0, VT);
647 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
648 }
else if (
Base.getValueType() != VT) {
650 assert(VT == MVT::i32 &&
Base.getValueType() == MVT::i64 &&
651 "Unexpected truncation");
659 Disp = CurDAG->getTargetConstant(AM.Disp,
SDLoc(
Base), VT);
662void SystemZDAGToDAGISel::getAddressOperands(
const SystemZAddressingMode &AM,
666 getAddressOperands(AM, VT,
Base, Disp);
669 if (!
Index.getNode())
671 Index = CurDAG->getRegister(0, VT);
674bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
677 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
678 if (!selectAddress(
Addr, AM))
681 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
685bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
688 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
689 if (!selectAddress(
Addr, AM) || AM.Index.getNode())
692 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp);
696bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm
Form,
697 SystemZAddressingMode::DispRange DR,
700 SystemZAddressingMode AM(
Form, DR);
701 if (!selectAddress(
Addr, AM))
704 getAddressOperands(AM,
Addr.getValueType(),
Base, Disp,
Index);
713 if (selectBDXAddr12Only(
Addr, Regs[0], Disp, Regs[1]) &&
714 Regs[0].getNode() && Regs[1].getNode()) {
715 for (
unsigned int I = 0;
I < 2; ++
I) {
723 Index.getOperand(1) == Elem) {
732bool SystemZDAGToDAGISel::detectOrAndInsertion(
SDValue &
Op,
740 auto *MaskNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1).getNode());
745 uint64_t AndMask = MaskNode->getZExtValue();
746 if (InsertMask & AndMask)
752 if (Used != (AndMask | InsertMask)) {
753 KnownBits Known = CurDAG->computeKnownBits(
Op.getOperand(0));
758 Op =
Op.getOperand(0);
762bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
765 if (RxSBG.Rotate != 0)
766 Mask = (
Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
768 if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
778 if (RxSBG.Rotate != 0)
779 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
780 return (Mask & RxSBG.Mask) != 0;
783bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG)
const {
785 unsigned Opcode =
N.getOpcode();
788 if (RxSBG.Opcode == SystemZ::RNSBG)
790 if (
N.getOperand(0).getValueSizeInBits() > 64)
792 uint64_t BitSize =
N.getValueSizeInBits();
794 if (!refineRxSBGMask(RxSBG, Mask))
796 RxSBG.Input =
N.getOperand(0);
800 if (RxSBG.Opcode == SystemZ::RNSBG)
803 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
809 if (!refineRxSBGMask(RxSBG, Mask)) {
813 KnownBits Known = CurDAG->computeKnownBits(Input);
815 if (!refineRxSBGMask(RxSBG, Mask))
823 if (RxSBG.Opcode != SystemZ::RNSBG)
826 auto *MaskNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
832 if (!refineRxSBGMask(RxSBG, Mask)) {
836 KnownBits Known = CurDAG->computeKnownBits(Input);
837 Mask &= ~Known.One.getZExtValue();
838 if (!refineRxSBGMask(RxSBG, Mask))
847 if (RxSBG.BitSize != 64 ||
N.getValueType() != MVT::i64)
849 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
853 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
854 RxSBG.Input =
N.getOperand(0);
860 RxSBG.Input =
N.getOperand(0);
864 if (RxSBG.Opcode != SystemZ::RNSBG) {
866 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
867 if (!refineRxSBGMask(RxSBG,
allOnes(InnerBitSize)))
870 RxSBG.Input =
N.getOperand(0);
878 unsigned BitSize =
N.getValueSizeInBits();
879 unsigned InnerBitSize =
N.getOperand(0).getValueSizeInBits();
883 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
884 RxSBG.Rotate += (BitSize - InnerBitSize);
889 RxSBG.Input =
N.getOperand(0);
894 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
898 uint64_t Count = CountNode->getZExtValue();
899 unsigned BitSize =
N.getValueSizeInBits();
900 if (Count < 1 || Count >= BitSize)
903 if (RxSBG.Opcode == SystemZ::RNSBG) {
910 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count) << Count))
914 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
915 RxSBG.Input =
N.getOperand(0);
921 auto *CountNode = dyn_cast<ConstantSDNode>(
N.getOperand(1).getNode());
925 uint64_t Count = CountNode->getZExtValue();
926 unsigned BitSize =
N.getValueSizeInBits();
927 if (Count < 1 || Count >= BitSize)
930 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode ==
ISD::SRA) {
938 if (!refineRxSBGMask(RxSBG,
allOnes(BitSize - Count)))
942 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
943 RxSBG.Input =
N.getOperand(0);
952 SDNode *
N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
958 if (
N.getValueType() == MVT::i32 && VT == MVT::i64)
959 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
960 DL, VT, getUNDEF(
DL, MVT::i64),
N);
961 if (
N.getValueType() == MVT::i64 && VT == MVT::i32)
962 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32,
DL, VT,
N);
963 assert(
N.getValueType() == VT &&
"Unexpected value types");
967bool SystemZDAGToDAGISel::tryRISBGZero(
SDNode *
N) {
969 EVT VT =
N->getValueType(0);
972 RxSBGOperands RISBG(SystemZ::RISBG,
SDValue(
N, 0));
974 while (expandRxSBG(RISBG))
981 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
986 if (Count == 1 &&
N->getOpcode() !=
ISD::AND)
992 if (RISBG.Rotate == 0) {
993 bool PreferAnd =
false;
999 else if (RISBG.Mask == 0xff ||
1000 RISBG.Mask == 0xffff ||
1001 RISBG.Mask == 0x7fffffff ||
1007 else if (
auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1008 if (
Load->getMemoryVT() == MVT::i32 &&
1011 RISBG.Mask == 0xffffff00 &&
1012 Subtarget->hasLoadAndZeroRightmostByte())
1022 if (
N !=
New.getNode()) {
1025 ReplaceNode(
N,
New.getNode());
1029 if (!
N->isMachineOpcode())
1035 unsigned Opcode = SystemZ::RISBG;
1037 if (Subtarget->hasMiscellaneousExtensions())
1038 Opcode = SystemZ::RISBGN;
1039 EVT OpcodeVT = MVT::i64;
1040 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1045 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1046 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1047 ((RISBG.End + RISBG.Rotate) & 63) >=
1048 ((RISBG.Start + RISBG.Rotate) & 63)) {
1049 Opcode = SystemZ::RISBMux;
1050 OpcodeVT = MVT::i32;
1055 getUNDEF(
DL, OpcodeVT),
1056 convertTo(
DL, OpcodeVT, RISBG.Input),
1057 CurDAG->getTargetConstant(RISBG.Start,
DL, MVT::i32),
1058 CurDAG->getTargetConstant(RISBG.End | 128,
DL, MVT::i32),
1059 CurDAG->getTargetConstant(RISBG.Rotate,
DL, MVT::i32)
1062 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, OpcodeVT, Ops), 0));
1063 ReplaceNode(
N,
New.getNode());
1067bool SystemZDAGToDAGISel::tryRxSBG(
SDNode *
N,
unsigned Opcode) {
1069 EVT VT =
N->getValueType(0);
1074 RxSBGOperands RxSBG[] = {
1075 RxSBGOperands(Opcode,
N->getOperand(0)),
1076 RxSBGOperands(Opcode,
N->getOperand(1))
1078 unsigned Count[] = { 0, 0 };
1079 for (
unsigned I = 0;
I < 2; ++
I)
1080 while (RxSBG[
I].Input->
hasOneUse() && expandRxSBG(RxSBG[
I]))
1092 if (Count[0] == 0 && Count[1] == 0)
1096 unsigned I = Count[0] > Count[1] ? 0 : 1;
1100 if (Opcode == SystemZ::ROSBG && (RxSBG[
I].Mask & 0xff) == 0)
1101 if (
auto *Load = dyn_cast<LoadSDNode>(Op0.
getNode()))
1102 if (
Load->getMemoryVT() == MVT::i8)
1107 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[
I].Mask)) {
1108 Opcode = SystemZ::RISBG;
1110 if (Subtarget->hasMiscellaneousExtensions())
1111 Opcode = SystemZ::RISBGN;
1115 convertTo(
DL, MVT::i64, Op0),
1116 convertTo(
DL, MVT::i64, RxSBG[
I].Input),
1117 CurDAG->getTargetConstant(RxSBG[
I].Start,
DL, MVT::i32),
1118 CurDAG->getTargetConstant(RxSBG[
I].
End,
DL, MVT::i32),
1119 CurDAG->getTargetConstant(RxSBG[
I].Rotate,
DL, MVT::i32)
1122 DL, VT,
SDValue(CurDAG->getMachineNode(Opcode,
DL, MVT::i64, Ops), 0));
1123 ReplaceNode(
N,
New.getNode());
1127void SystemZDAGToDAGISel::splitLargeImmediate(
unsigned Opcode,
SDNode *
Node,
1130 EVT VT =
Node->getValueType(0);
1150 SelectCode(
Upper.getNode());
1151 Upper = Handle.getValue();
1157 ReplaceNode(
Node,
Or.getNode());
1159 SelectCode(
Or.getNode());
1162void SystemZDAGToDAGISel::loadVectorConstant(
1169 EVT VT =
Node->getValueType(0);
1172 for (
unsigned OpVal : VCI.
OpVals)
1173 Ops.
push_back(CurDAG->getTargetConstant(OpVal,
DL, MVT::i32));
1177 ReplaceNode(
Node,
Op.getNode());
1181 SelectCode(BitCast.
getNode());
1183 unsigned SubRegIdx =
1184 (VT.
getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1186 Node, CurDAG->getTargetExtractSubreg(SubRegIdx,
DL, VT,
Op).getNode());
1188 SelectCode(
Op.getNode());
1195 SDValue CP = CurDAG->getTargetConstantPool(
1197 TLI->getPointerTy(CurDAG->getDataLayout()));
1199 EVT PtrVT =
CP.getValueType();
1201 SDValue(CurDAG->getMachineNode(SystemZ::LARL,
DL, PtrVT, CP), 0),
1202 CurDAG->getTargetConstant(0,
DL, PtrVT),
1203 CurDAG->getRegister(0, PtrVT),
1204 CurDAG->getEntryNode()
1206 ResNode = CurDAG->getMachineNode(SystemZ::VL,
DL, VT, MVT::Other, Ops);
1216 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
1220bool SystemZDAGToDAGISel::tryGather(
SDNode *
N,
unsigned Opcode) {
1222 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1226 unsigned Elem = ElemN->getZExtValue();
1227 EVT VT =
N->getValueType(0);
1231 auto *
Load = dyn_cast<LoadSDNode>(
N->getOperand(1));
1232 if (!Load || !
Load->hasNUsesOfValue(1, 0))
1234 if (
Load->getMemoryVT().getSizeInBits() !=
1235 Load->getValueType(0).getSizeInBits())
1239 if (!selectBDVAddr12Only(
Load->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1246 CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
Load->getChain()
1248 SDNode *Res = CurDAG->getMachineNode(Opcode,
DL, VT, MVT::Other, Ops);
1250 ReplaceNode(
N, Res);
1254bool SystemZDAGToDAGISel::tryScatter(
StoreSDNode *Store,
unsigned Opcode) {
1258 if (
Store->getMemoryVT().getSizeInBits() !=
Value.getValueSizeInBits())
1262 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1268 unsigned Elem = ElemN->getZExtValue();
1273 if (!selectBDVAddr12Only(
Store->getBasePtr(), ElemV,
Base, Disp,
Index) ||
1279 Vec,
Base, Disp,
Index, CurDAG->getTargetConstant(Elem,
DL, MVT::i32),
1282 ReplaceNode(Store, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
1310 LoadNode = cast<LoadSDNode>(Load);
1313 if (!Load.hasOneUse())
1325 bool ChainCheck =
false;
1326 if (Chain == Load.getValue(1)) {
1333 const unsigned int Max = 1024;
1336 if (
Op == Load.getValue(1)) {
1349 if (
Op.getNode() != LoadNode)
1360 MVT::Other, ChainOps);
1377bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(
SDNode *
Node) {
1387 unsigned NewOpc = 0;
1388 bool NegateOperand =
false;
1393 NegateOperand =
true;
1396 if (MemVT == MVT::i32)
1397 NewOpc = SystemZ::ASI;
1398 else if (MemVT == MVT::i64)
1399 NewOpc = SystemZ::AGSI;
1404 NegateOperand =
true;
1407 if (MemVT == MVT::i32)
1408 NewOpc = SystemZ::ALSI;
1409 else if (MemVT == MVT::i64)
1410 NewOpc = SystemZ::ALGSI;
1423 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1426 auto OperandV = OperandC->getAPIntValue();
1428 OperandV = -OperandV;
1429 if (OperandV.getSignificantBits() > 8)
1431 Operand = CurDAG->getTargetConstant(OperandV,
DL, MemVT);
1437 SDValue Ops[] = {
Base, Disp, Operand, InputChain };
1439 CurDAG->getMachineNode(NewOpc,
DL, MVT::i32, MVT::Other, Ops);
1440 CurDAG->setNodeMemRefs(
1445 CurDAG->RemoveDeadNode(
Node);
1449bool SystemZDAGToDAGISel::canUseBlockOperation(
StoreSDNode *Store,
1452 if (
Load->getMemoryVT() !=
Store->getMemoryVT())
1456 if (
Load->isVolatile() ||
Store->isVolatile())
1460 if (
Load->isInvariant() &&
Load->isDereferenceable())
1464 const Value *V1 =
Load->getMemOperand()->getValue();
1471 int64_t End1 =
Load->getSrcValueOffset() +
Size;
1472 int64_t End2 =
Store->getSrcValueOffset() +
Size;
1473 if (V1 == V2 && End1 == End2)
1480bool SystemZDAGToDAGISel::storeLoadCanUseMVC(
SDNode *
N)
const {
1481 auto *
Store = cast<StoreSDNode>(
N);
1482 auto *
Load = cast<LoadSDNode>(
Store->getValue());
1496 return canUseBlockOperation(Store, Load);
1499bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(
SDNode *
N,
1501 auto *StoreA = cast<StoreSDNode>(
N);
1502 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 -
I));
1503 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(
I));
1504 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1505 canUseBlockOperation(StoreA, LoadB);
1508bool SystemZDAGToDAGISel::storeLoadIsAligned(
SDNode *
N)
const {
1510 auto *MemAccess = cast<LSBaseSDNode>(
N);
1511 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1514 assert(MMO &&
"Expected a memory operand.");
1517 if (MemAccess->getAlign().value() < StoreSize ||
1518 !MemAccess->getOffset().isUndef())
1527 if ((PSV->isGOT() || PSV->isConstantPool()))
1533 dyn_cast<GlobalAddressSDNode>(
BasePtr.getOperand(0))) {
1535 if (GA->getOffset() % StoreSize != 0)
1548void SystemZDAGToDAGISel::Select(
SDNode *
Node) {
1550 if (
Node->isMachineOpcode()) {
1552 Node->setNodeId(-1);
1556 unsigned Opcode =
Node->getOpcode();
1560 if (tryRxSBG(
Node, SystemZ::ROSBG))
1566 if (tryRxSBG(
Node, SystemZ::RXSBG))
1573 if (
Node->getValueType(0) == MVT::i64 &&
1575 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Node->getOperand(1))) {
1576 uint64_t Val = Op1->getZExtValue();
1579 if (Subtarget->hasMiscellaneousExtensions3()) {
1580 unsigned ChildOpcode =
Node->getOperand(0).getOpcode();
1589 auto Op0 =
Node->getOperand(0);
1590 if (
auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->
getOperand(1)))
1591 if (Op0Op1->getZExtValue() == (
uint64_t)-1)
1596 if (Opcode ==
ISD::XOR && Op1->isAllOnes())
1599 splitLargeImmediate(Opcode,
Node,
Node->getOperand(0),
1608 if (tryRxSBG(
Node, SystemZ::RNSBG))
1615 if (tryRISBGZero(
Node))
1620 if (
Node->getValueType(0) == MVT::i128) {
1625 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1629 MVT::v16i8, Ops), 0);
1633 ReplaceNode(
Node, ResNode);
1634 SelectCode(Src.getNode());
1635 SelectCode(ResNode);
1643 if (
Node->getValueType(0) == MVT::i64) {
1651 if (
Node->getValueType(0) == MVT::i128) {
1652 const APInt &Val =
Node->getAsAPIntVal();
1655 loadVectorConstant(VCI,
Node);
1659 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128,
SDLoc(
Node));
1660 ReplaceNode(
Node, ResNode);
1671 (Subtarget->hasLoadStoreOnCond2() &&
1672 Node->getValueType(0).isInteger() &&
1673 Node->getValueType(0).getSizeInBits() <= 64 &&
1675 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1677 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1683 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1687 CurDAG->UpdateNodeOperands(
Node, Op1, Op0, CCValid, CCMask, Op4);
1688 if (UpdatedNode !=
Node) {
1690 ReplaceNode(
Node, UpdatedNode);
1698 EVT VT =
Node->getValueType(0);
1700 if (ElemBitSize == 32) {
1701 if (tryGather(
Node, SystemZ::VGEF))
1703 }
else if (ElemBitSize == 64) {
1704 if (tryGather(
Node, SystemZ::VGEG))
1711 auto *BVN = cast<BuildVectorSDNode>(
Node);
1714 loadVectorConstant(VCI,
Node);
1722 if (
Imm.isZero() ||
Imm.isNegZero())
1727 loadVectorConstant(VCI,
Node);
1732 if (tryFoldLoadStoreIntoMemOperand(
Node))
1735 unsigned ElemBitSize =
Store->getValue().getValueSizeInBits();
1736 if (ElemBitSize == 32) {
1737 if (tryScatter(Store, SystemZ::VSCEF))
1739 }
else if (ElemBitSize == 64) {
1740 if (tryScatter(Store, SystemZ::VSCEG))
1750bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1752 std::vector<SDValue> &OutOps) {
1753 SystemZAddressingMode::AddrForm
Form;
1754 SystemZAddressingMode::DispRange DispRange;
1757 switch(ConstraintID) {
1760 case InlineAsm::ConstraintCode::i:
1761 case InlineAsm::ConstraintCode::Q:
1762 case InlineAsm::ConstraintCode::ZQ:
1764 Form = SystemZAddressingMode::FormBD;
1765 DispRange = SystemZAddressingMode::Disp12Only;
1767 case InlineAsm::ConstraintCode::R:
1768 case InlineAsm::ConstraintCode::ZR:
1770 Form = SystemZAddressingMode::FormBDXNormal;
1771 DispRange = SystemZAddressingMode::Disp12Only;
1773 case InlineAsm::ConstraintCode::S:
1774 case InlineAsm::ConstraintCode::ZS:
1776 Form = SystemZAddressingMode::FormBD;
1777 DispRange = SystemZAddressingMode::Disp20Only;
1779 case InlineAsm::ConstraintCode::T:
1780 case InlineAsm::ConstraintCode::m:
1781 case InlineAsm::ConstraintCode::o:
1782 case InlineAsm::ConstraintCode::p:
1783 case InlineAsm::ConstraintCode::ZT:
1788 Form = SystemZAddressingMode::FormBDXNormal;
1789 DispRange = SystemZAddressingMode::Disp20Only;
1795 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1797 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(),
DL, MVT::i32);
1804 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1812 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1817 OutOps.push_back(
Base);
1818 OutOps.push_back(Disp);
1819 OutOps.push_back(
Index);
1834 if (!
N.hasOneUse() || !
U->hasOneUse())
1841 SDNode *CCUser = *
U->use_begin();
1842 SDNode *CCRegUser =
nullptr;
1844 cast<RegisterSDNode>(CCUser->
getOperand(1))->getReg() == SystemZ::CC) {
1845 for (
auto *U : CCUser->
uses()) {
1846 if (CCRegUser ==
nullptr)
1848 else if (CCRegUser != U)
1852 if (CCRegUser ==
nullptr)
1865 if (!IsLegalToFold(
N, U, CCRegUser, OptLevel,
false))
1875struct IPMConversion {
1876 IPMConversion(
unsigned xorValue, int64_t addValue,
unsigned bit)
1877 : XORValue(xorValue), AddValue(addValue),
Bit(bit) {}
1953 auto *TrueOp = dyn_cast<ConstantSDNode>(
Node->getOperand(0));
1954 auto *FalseOp = dyn_cast<ConstantSDNode>(
Node->getOperand(1));
1955 if (!TrueOp || !FalseOp)
1957 if (FalseOp->getZExtValue() != 0)
1959 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1962 auto *CCValidOp = dyn_cast<ConstantSDNode>(
Node->getOperand(2));
1963 auto *CCMaskOp = dyn_cast<ConstantSDNode>(
Node->getOperand(3));
1964 if (!CCValidOp || !CCMaskOp)
1966 int CCValid = CCValidOp->getZExtValue();
1967 int CCMask = CCMaskOp->getZExtValue();
1976 CurDAG->getConstant(
IPM.XORValue,
DL, MVT::i32));
1980 CurDAG->getConstant(
IPM.AddValue,
DL, MVT::i32));
1982 EVT VT =
Node->getValueType(0);
1983 if (VT == MVT::i32 &&
IPM.Bit == 31) {
1985 Result = CurDAG->getNode(ShiftOp,
DL, MVT::i32, Result,
1986 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
1991 if (TrueOp->getSExtValue() == 1) {
1994 CurDAG->getConstant(
IPM.Bit,
DL, MVT::i32));
1996 CurDAG->getConstant(1,
DL, VT));
2002 CurDAG->getConstant(ShlAmt,
DL, MVT::i32));
2004 CurDAG->getConstant(SraAmt,
DL, MVT::i32));
2011void SystemZDAGToDAGISel::PreprocessISelDAG() {
2014 if (Subtarget->hasLoadStoreOnCond2())
2017 bool MadeChange =
false;
2020 E = CurDAG->allnodes_end();
2027 switch (
N->getOpcode()) {
2030 Res = expandSelectBoolean(
N);
2035 LLVM_DEBUG(
dbgs() <<
"SystemZ DAG preprocessing replacing:\nOld: ");
2041 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
2047 CurDAG->RemoveDeadNodes();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
@ MOLoad
The memory access reads data.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static IntegerType * getInt128Ty(LLVMContext &C)
LLVM Value Representation.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ BSWAP
Byte Swap and Counting operators.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
static bool isImmHF(uint64_t Val)
static bool isImmLF(uint64_t Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)