23#include "llvm/Config/llvm-config.h"
28#include "llvm/IR/IntrinsicsX86.h"
39#define DEBUG_TYPE "x86-isel"
40#define PASS_NAME "X86 DAG->DAG Instruction Selection"
42STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
45 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
49 "x86-promote-anyext-load",
cl::init(
true),
61 struct X86ISelAddressMode {
69 int Base_FrameIndex = 0;
78 const char *ES =
nullptr;
83 bool NegateIndex =
false;
85 X86ISelAddressMode() =
default;
87 bool hasSymbolicDisplacement()
const {
88 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
89 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
92 bool hasBaseOrIndexReg()
const {
99 if (
BaseType != RegBase)
return false;
101 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
102 return RegNode->getReg() == X86::RIP;
111#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
113 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
114 dbgs() <<
"Base_Reg ";
120 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
121 dbgs() <<
" Scale " << Scale <<
'\n'
129 dbgs() <<
" Disp " << Disp <<
'\n'
151 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
171 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
201#include "X86GenDAGISel.inc"
206 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
207 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
208 bool AllowSegmentRegForX32 =
false);
209 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
210 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
211 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
215 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
217 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
248 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
256 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
261 std::vector<SDValue> &OutOps)
override;
263 void emitSpecialCodeForMain();
265 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
269 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
270 Base = CurDAG->getTargetFrameIndex(
271 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
272 else if (AM.Base_Reg.getNode())
275 Base = CurDAG->getRegister(0, VT);
277 Scale = getI8Imm(AM.Scale,
DL);
279#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
281 if (AM.NegateIndex) {
289 if (AM.IndexReg.getNode())
292 Index = CurDAG->getRegister(0, VT);
297 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
301 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
302 AM.Disp, AM.SymbolFlags);
304 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
305 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
306 }
else if (AM.MCSym) {
307 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
308 assert(AM.SymbolFlags == 0 &&
"oo");
309 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
310 }
else if (AM.JT != -1) {
311 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
312 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
313 }
else if (AM.BlockAddr)
314 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
317 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
319 if (AM.Segment.getNode())
320 Segment = AM.Segment;
322 Segment = CurDAG->getRegister(0, MVT::i16);
331 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
337 if (!CurDAG->shouldOptForSize())
347 if (
User->isMachineOpcode()) {
370 auto *
C = dyn_cast<ConstantSDNode>(
N);
371 if (
C && isInt<8>(
C->getSExtValue()))
391 (RegNode = dyn_cast_or_null<RegisterSDNode>(
393 if ((RegNode->
getReg() == X86::ESP) ||
394 (RegNode->
getReg() == X86::RSP))
403 return (UseCount > 1);
408 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
413 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
418 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
423 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
425 MVT VecVT =
N->getOperand(0).getSimpleValueType();
431 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
433 MVT VecVT =
N->getSimpleValueType(0);
437 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
439 assert(VecWidth == 128 &&
"Unexpected vector width");
441 MVT VecVT =
N->getSimpleValueType(0);
443 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
446 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
451 MVT VT =
N->getSimpleValueType(0);
454 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
456 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
457 if (VT == MVT::i64) {
459 CurDAG->getMachineNode(
460 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
461 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
462 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
467 unsigned Opcode =
N->getOpcode();
469 "Unexpected opcode for SBB materialization");
470 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
472 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
473 N->getOperand(FlagOpIndex),
SDValue());
477 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
478 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
479 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
481 CurDAG->getMachineNode(Opc, dl, VTs,
482 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
488 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
490 const APInt &Val =
N->getConstantOperandAPInt(1);
495 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
496 return Mask.countr_one() >= Width;
502 SDNode *getGlobalBaseReg();
513 return Subtarget->getInstrInfo();
526 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
530 if (!
N->isNonTemporal())
533 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
535 if (
N->getAlign().value() < StoreSize)
544 return Subtarget->hasSSE41();
546 return Subtarget->hasAVX2();
548 return Subtarget->hasAVX512();
552 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
555 bool shrinkAndImmediate(
SDNode *
N);
556 bool isMaskZeroExtended(
SDNode *
N)
const;
557 bool tryShiftAmountMod(
SDNode *
N);
558 bool tryShrinkShlLogicImm(
SDNode *
N);
564 bool tryMatchBitSelect(
SDNode *
N);
566 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
568 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
572 bool tryOptimizeRem8Extend(
SDNode *
N);
574 bool onlyUsesZeroFlag(
SDValue Flags)
const;
575 bool hasNoSignFlagUses(
SDValue Flags)
const;
576 bool hasNoCarryFlagUses(
SDValue Flags)
const;
585 ID,
std::make_unique<X86DAGToDAGISel>(tm, OptLevel)) {}
589char X86DAGToDAGISelLegacy::ID = 0;
596 unsigned Opcode =
N->getOpcode();
603 EVT OpVT =
N->getOperand(0).getValueType();
607 OpVT =
N->getOperand(1).getValueType();
609 return Subtarget->hasVLX();
623bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
636 if (OptLevel == CodeGenOptLevel::None)
646 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
651 switch (
U->getOpcode()) {
677 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
678 if (
Imm->getAPIntValue().isSignedIntN(8))
687 Imm->getAPIntValue().getBitWidth() == 64 &&
688 Imm->getAPIntValue().isIntN(32))
695 (
Imm->getAPIntValue() == UINT8_MAX ||
696 Imm->getAPIntValue() == UINT16_MAX ||
697 Imm->getAPIntValue() == UINT32_MAX))
703 (-
Imm->getAPIntValue()).isSignedIntN(8))
707 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
708 hasNoCarryFlagUses(
SDValue(U, 1)))
733 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
737 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
745 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
746 if (
C &&
C->getSExtValue() == -2)
751 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
752 if (
C &&
C->getSExtValue() == -2)
766 if (isa<ConstantSDNode>(
U->getOperand(1)))
787bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
790 "Unexpected opcode!");
795 return N->getOperand(1).hasOneUse();
804 if (Chain.
getNode() == Load.getNode())
808 "Unexpected chain operand");
822 Load.getOperand(1), Load.getOperand(2));
826 Ops.
append(Call->op_begin() + 1, Call->op_end());
840 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
842 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
860 if (isa<MemSDNode>(Chain.
getNode()) &&
861 cast<MemSDNode>(Chain.
getNode())->writeMem())
867 Callee.getValue(1).hasOneUse())
875 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
878 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
879 0x65, 0x66, 0x67, 0xf0, 0xf2};
882 uint8_t Byte = (Imm >> i) & 0xFF;
894 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
897void X86DAGToDAGISel::PreprocessISelDAG() {
898 bool MadeChange =
false;
900 E = CurDAG->allnodes_end();
I != E; ) {
919 MVT VT =
N->getSimpleValueType(0);
920 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
921 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
926 "cf-protection-branch");
929 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
930 Complement = CurDAG->getNOT(dl, Complement, VT);
932 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
942 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
944 N->getOperand(0),
N->getOperand(1));
946 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
970 auto mayPreventLoadFold = [&]() {
972 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
973 !
N->getOperand(1).hasOneUse();
976 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
982 MVT VT =
N->getSimpleValueType(0);
990 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
992 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
999 switch (
N->getOpcode()) {
1001 MVT VT =
N->getSimpleValueType(0);
1003 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1010 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1013 CurDAG->getIntPtrConstant(
Index, dl));
1016 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1025 MVT VT =
N->getSimpleValueType(0);
1027 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1029 auto *MemNode = cast<MemSDNode>(
N);
1031 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1032 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1033 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1035 MemNode->getMemOperand());
1038 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1041 CurDAG->getIntPtrConstant(
Index, dl));
1045 CurDAG->ReplaceAllUsesWith(
N, To);
1056 auto *Ld = cast<LoadSDNode>(
N);
1057 MVT VT =
N->getSimpleValueType(0);
1065 SDValue Chain = Ld->getChain();
1067 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1068 MVT UserVT =
User->getSimpleValueType(0);
1070 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1071 !
User->hasAnyUseOfValue(1) &&
1085 CurDAG->getIntPtrConstant(0, dl));
1086 SDValue Res = CurDAG->getBitcast(VT, Extract);
1090 CurDAG->ReplaceAllUsesWith(
N, To);
1099 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1100 if (EleVT == MVT::i1)
1103 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1104 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1105 "We can't replace VSELECT with BLENDV in vXi16!");
1107 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1110 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1111 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1114 N->getOperand(0),
N->getOperand(1),
1118 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1131 if (!
N->getSimpleValueType(0).isVector())
1135 switch (
N->getOpcode()) {
1145 if (
N->isStrictFPOpcode())
1147 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1148 {
N->getOperand(0),
N->getOperand(1)});
1151 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1154 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1164 if (!
N->getValueType(0).isVector())
1168 switch (
N->getOpcode()) {
1174 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1175 N->getOperand(0),
N->getOperand(1));
1177 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1186 if (!
N->getValueType(0).isVector())
1190 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1192 "Unexpected opcode for mask vector!");
1200 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1203 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1223 switch (
N->getOpcode()) {
1239 bool IsStrict =
N->isStrictFPOpcode();
1243 {
N->getValueType(0), MVT::Other},
1244 {
N->getOperand(0),
N->getOperand(1),
1245 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1249 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1251 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1262 MVT VT =
N->getSimpleValueType(0);
1263 if (VT.
isVector() || VT == MVT::f128)
1266 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1267 : VT == MVT::f32 ? MVT::v4f32
1277 if (Subtarget->hasSSE2()) {
1282 switch (
N->getOpcode()) {
1289 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1292 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1295 CurDAG->getIntPtrConstant(0, dl));
1297 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1304 if (OptLevel != CodeGenOptLevel::None &&
1307 !Subtarget->useIndirectThunkCalls() &&
1308 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1310 (Subtarget->is64Bit() ||
1311 !getTargetMachine().isPositionIndependent())))) {
1350 switch (
N->getOpcode()) {
1355 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1356 MVT DstVT =
N->getSimpleValueType(0);
1368 if (SrcIsSSE && DstIsSSE)
1371 if (!SrcIsSSE && !DstIsSSE) {
1376 if (
N->getConstantOperandVal(1))
1384 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1385 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1393 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1395 MemTmp, MPI, MemVT);
1402 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1411 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1412 MVT DstVT =
N->getSimpleValueType(0);
1424 if (SrcIsSSE && DstIsSSE)
1427 if (!SrcIsSSE && !DstIsSSE) {
1432 if (
N->getConstantOperandVal(2))
1440 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1441 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1451 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1452 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1456 if (
N->getFlags().hasNoFPExcept()) {
1458 Flags.setNoFPExcept(
true);
1459 Store->setFlags(Flags);
1462 assert(SrcVT == MemVT &&
"Unexpected VT!");
1463 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1468 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1470 Result = CurDAG->getMemIntrinsicNode(
1473 if (
N->getFlags().hasNoFPExcept()) {
1475 Flags.setNoFPExcept(
true);
1479 assert(DstVT == MemVT &&
"Unexpected VT!");
1480 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1488 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1502 CurDAG->RemoveDeadNodes();
1506bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1507 unsigned Opc =
N->getMachineOpcode();
1508 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1509 Opc != X86::MOVSX64rr8)
1521 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1522 : X86::MOVSX32rr8_NOREX;
1527 if (Opc == X86::MOVSX64rr8) {
1532 ReplaceUses(
N, Extend);
1541void X86DAGToDAGISel::PostprocessISelDAG() {
1543 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1548 bool MadeChange =
false;
1549 while (Position != CurDAG->allnodes_begin()) {
1552 if (
N->use_empty() || !
N->isMachineOpcode())
1555 if (tryOptimizeRem8Extend(
N)) {
1560 unsigned Opc =
N->getMachineOpcode();
1571 case X86::CTEST16rr:
1572 case X86::CTEST32rr:
1573 case X86::CTEST64rr: {
1574 auto &Op0 =
N->getOperand(0);
1579#define CASE_ND(OP) \
1582 switch (
And.getMachineOpcode()) {
1589 if (
And->hasAnyUseOfValue(1))
1592 Ops[0] =
And.getOperand(0);
1593 Ops[1] =
And.getOperand(1);
1595 CurDAG->getMachineNode(Opc,
SDLoc(
N), MVT::i32, Ops);
1596 ReplaceUses(
N,
Test);
1604 if (
And->hasAnyUseOfValue(1))
1607 bool IsCTESTCC = X86::isCTESTCC(Opc);
1608#define FROM_TO(A, B) \
1609 CASE_ND(A) NewOpc = IsCTESTCC ? X86::C##B : X86::B; \
1611 switch (
And.getMachineOpcode()) {
1621 And.getOperand(3),
And.getOperand(4),
1622 And.getOperand(5),
And.getOperand(0)};
1635 NewOpc,
SDLoc(
N), MVT::i32, MVT::Other, Ops);
1636 CurDAG->setNodeMemRefs(
1637 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1649 case X86::KORTESTBrr:
1650 case X86::KORTESTWrr:
1651 case X86::KORTESTDrr:
1652 case X86::KORTESTQrr: {
1654 if (Op0 !=
N->getOperand(1) || !
N->isOnlyUserOf(Op0.
getNode()) ||
1669#define FROM_TO(A, B) \
1681 if (NewOpc == X86::KTESTWrr && !Subtarget->hasDQI())
1686 ReplaceUses(
N, KTest);
1691 case TargetOpcode::SUBREG_TO_REG: {
1692 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1693 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1710 CASE(VMOVAPDZ128rr)
CASE(VMOVUPDZ128rr)
1711 CASE(VMOVAPSZ128rr)
CASE(VMOVUPSZ128rr)
1712 CASE(VMOVDQA32Z128rr)
CASE(VMOVDQU32Z128rr)
1713 CASE(VMOVDQA64Z128rr)
CASE(VMOVDQU64Z128rr)
1714 CASE(VMOVAPDZ256rr)
CASE(VMOVUPDZ256rr)
1715 CASE(VMOVAPSZ256rr)
CASE(VMOVUPSZ256rr)
1716 CASE(VMOVDQA32Z256rr)
CASE(VMOVDQU32Z256rr)
1717 CASE(VMOVDQA64Z256rr)
CASE(VMOVDQU64Z256rr)
1722 if (!
In.isMachineOpcode() ||
1723 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1728 uint64_t TSFlags = getInstrInfo()->get(
In.getMachineOpcode()).TSFlags;
1736 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1743 CurDAG->RemoveDeadNodes();
1748void X86DAGToDAGISel::emitSpecialCodeForMain() {
1749 if (Subtarget->isTargetCygMing()) {
1751 auto &
DL = CurDAG->getDataLayout();
1754 CLI.setChain(CurDAG->getRoot())
1756 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1760 CurDAG->setRoot(
Result.second);
1764void X86DAGToDAGISel::emitFunctionEntryCode() {
1767 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1768 emitSpecialCodeForMain();
1778 return isInt<31>(Val);
1782 X86ISelAddressMode &AM) {
1787 int64_t Val = AM.Disp +
Offset;
1790 if (Val != 0 && (AM.ES || AM.MCSym))
1794 if (Subtarget->is64Bit()) {
1797 AM.hasSymbolicDisplacement()))
1801 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1820 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1821 !AM.hasBaseOrIndexReg())
1828bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1829 bool AllowSegmentRegForX32) {
1841 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1842 !IndirectTlsSegRefs &&
1843 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1844 Subtarget->isTargetFuchsia())) {
1845 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1847 switch (
N->getPointerInfo().getAddrSpace()) {
1849 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1852 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1865bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1868 if (AM.hasSymbolicDisplacement())
1871 bool IsRIPRelTLS =
false;
1889 if (IsRIPRel && AM.hasBaseOrIndexReg())
1893 X86ISelAddressMode Backup = AM;
1897 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1898 AM.GV =
G->getGlobal();
1899 AM.SymbolFlags =
G->getTargetFlags();
1901 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1902 AM.CP =
CP->getConstVal();
1903 AM.Alignment =
CP->getAlign();
1904 AM.SymbolFlags =
CP->getTargetFlags();
1906 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1907 AM.ES = S->getSymbol();
1908 AM.SymbolFlags = S->getTargetFlags();
1909 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1910 AM.MCSym = S->getMCSymbol();
1911 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1912 AM.JT = J->getIndex();
1913 AM.SymbolFlags = J->getTargetFlags();
1914 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1915 AM.BlockAddr = BA->getBlockAddress();
1916 AM.SymbolFlags = BA->getTargetFlags();
1917 Offset = BA->getOffset();
1922 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1923 TM.isLargeGlobalValue(AM.GV)) {
1928 if (foldOffsetIntoAddress(
Offset, AM)) {
1934 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1942bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1943 if (matchAddressRecursively(
N, AM, 0))
1950 if (Subtarget->isTarget64BitILP32() &&
1951 AM.BaseType == X86ISelAddressMode::RegBase &&
1952 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1953 SDValue Save_Base_Reg = AM.Base_Reg;
1954 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1956 if (matchLoadInAddress(LoadN, AM,
true))
1957 AM.Base_Reg = Save_Base_Reg;
1963 if (AM.Scale == 2 &&
1964 AM.BaseType == X86ISelAddressMode::RegBase &&
1965 AM.Base_Reg.getNode() ==
nullptr) {
1966 AM.Base_Reg = AM.IndexReg;
1973 (!AM.GV || !
TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
1974 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
1975 AM.Base_Reg.getNode() ==
nullptr && AM.IndexReg.getNode() ==
nullptr &&
1977 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1983bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1989 X86ISelAddressMode Backup = AM;
1990 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1991 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1996 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1998 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
2005 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2006 !AM.Base_Reg.getNode() &&
2007 !AM.IndexReg.getNode()) {
2008 N = Handle.getValue();
2009 AM.Base_Reg =
N.getOperand(0);
2010 AM.IndexReg =
N.getOperand(1);
2014 N = Handle.getValue();
2024 if (
N->getNodeId() == -1 ||
2044 X86ISelAddressMode &AM) {
2051 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2052 Mask != (0xffu << ScaleLog))
2055 MVT XVT =
X.getSimpleValueType();
2056 MVT VT =
N.getSimpleValueType();
2081 AM.Scale = (1 << ScaleLog);
2089 X86ISelAddressMode &AM) {
2095 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2100 bool FoundAnyExtend =
false;
2104 FoundAnyExtend =
true;
2122 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2125 MVT VT =
N.getSimpleValueType();
2127 if (FoundAnyExtend) {
2148 AM.Scale = 1 << ShiftAmt;
2149 AM.IndexReg = NewAnd;
2183 X86ISelAddressMode &AM) {
2189 unsigned MaskIdx, MaskLen;
2192 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2198 unsigned AMShiftAmt = MaskIdx;
2202 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2206 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2207 if (MaskLZ < ScaleDown)
2209 MaskLZ -= ScaleDown;
2217 bool ReplacingAnyExtend =
false;
2219 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2220 X.getOperand(0).getSimpleValueType().getSizeInBits();
2223 X =
X.getOperand(0);
2224 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2225 ReplacingAnyExtend =
true;
2227 APInt MaskedHighBits =
2234 MVT VT =
N.getSimpleValueType();
2235 if (ReplacingAnyExtend) {
2236 assert(
X.getValueType() != VT);
2243 MVT XVT =
X.getSimpleValueType();
2264 AM.Scale = 1 << AMShiftAmt;
2265 AM.IndexReg = NewExt;
2275 X86ISelAddressMode &AM,
2283 if (!Subtarget.hasTBM() &&
2284 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2288 unsigned MaskIdx, MaskLen;
2296 unsigned AMShiftAmt = MaskIdx;
2300 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2302 MVT XVT =
X.getSimpleValueType();
2303 MVT VT =
N.getSimpleValueType();
2328 AM.Scale = 1 << AMShiftAmt;
2329 AM.IndexReg = NewExt;
2336 X86ISelAddressMode &AM,
2338 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2339 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2340 "Illegal index scale");
2346 EVT VT =
N.getValueType();
2347 unsigned Opc =
N.getOpcode();
2350 if (CurDAG->isBaseWithConstantOffset(
N)) {
2351 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2353 if (!foldOffsetIntoAddress(
Offset, AM))
2354 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2358 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2359 if (AM.Scale <= 4) {
2361 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2367 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2368 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2369 if ((AM.Scale * ScaleAmt) <= 8) {
2370 AM.Scale *= ScaleAmt;
2371 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2379 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2381 if (CurDAG->isBaseWithConstantOffset(Src)) {
2382 SDValue AddSrc = Src.getOperand(0);
2383 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2385 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2387 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2393 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2394 CurDAG->RemoveDeadNode(
N.getNode());
2406 unsigned SrcOpc = Src.getOpcode();
2407 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2408 CurDAG->isADDLike(Src,
true)) &&
2410 if (CurDAG->isBaseWithConstantOffset(Src)) {
2411 SDValue AddSrc = Src.getOperand(0);
2413 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2424 if ((AM.Scale * ScaleAmt) <= 8 &&
2426 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2427 AM.Scale *= ScaleAmt;
2428 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2437 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2439 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2443 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2444 CurDAG->RemoveDeadNode(
N.getNode());
2445 return Res ? Res : ExtSrc;
2455bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2459 dbgs() <<
"MatchAddress: ";
2464 return matchAddressBase(
N, AM);
2469 if (AM.isRIPRelative()) {
2473 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2476 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2477 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2482 switch (
N.getOpcode()) {
2485 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2486 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2488 AM.MCSym = ESNode->getMCSymbol();
2494 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2495 if (!foldOffsetIntoAddress(Val, AM))
2502 if (!matchWrapper(
N, AM))
2507 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2512 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2513 AM.Base_Reg.getNode() ==
nullptr &&
2515 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2516 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2522 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2525 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2526 unsigned Val = CN->getZExtValue();
2531 if (Val == 1 || Val == 2 || Val == 3) {
2533 AM.Scale = 1 << Val;
2534 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2542 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2546 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2547 "Unexpected value size!");
2556 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2557 !isa<ConstantSDNode>(
And.getOperand(1)))
2559 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2571 if (
N.getResNo() != 0)
break;
2576 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2577 AM.Base_Reg.getNode() ==
nullptr &&
2578 AM.IndexReg.getNode() ==
nullptr) {
2579 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2580 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2581 CN->getZExtValue() == 9) {
2582 AM.Scale =
unsigned(CN->getZExtValue())-1;
2593 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2594 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2595 if (foldOffsetIntoAddress(Disp, AM))
2596 Reg =
N.getOperand(0);
2598 Reg =
N.getOperand(0);
2601 AM.IndexReg = AM.Base_Reg =
Reg;
2620 X86ISelAddressMode Backup = AM;
2621 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2622 N = Handle.getValue();
2626 N = Handle.getValue();
2628 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2643 RHS.getOperand(0).getValueType() == MVT::i32))
2647 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2648 !AM.Base_Reg.getNode()->hasOneUse()) ||
2649 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2653 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2654 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2655 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2667 AM.NegateIndex =
true;
2675 if (!CurDAG->isADDLike(
N))
2679 if (!matchAdd(
N, AM,
Depth))
2688 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2692 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2693 "Unexpected value size!");
2695 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2698 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2727 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2737 AM.IndexReg =
Index;
2743 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2744 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2745 Mask = MaskC->getAPIntValue();
2746 Src = Src.getOperand(0);
2749 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse() &&
N->hasOneUse()) {
2751 SDValue ShlSrc = Src.getOperand(0);
2752 SDValue ShlAmt = Src.getOperand(1);
2753 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2756 unsigned ShAmtV = ShAmtC->getZExtValue();
2764 if (!Src->getFlags().hasNoUnsignedWrap() &&
2765 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2773 MVT VT =
N.getSimpleValueType();
2777 if (!
Mask.isAllOnes()) {
2778 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2780 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2787 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2788 CurDAG->RemoveDeadNode(
N.getNode());
2791 AM.Scale = 1 << ShAmtV;
2795 AM.IndexReg = matchIndexRecursively(Zext, AM,
Depth + 1);
2799 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2802 Src.getOperand(0), AM))
2807 Src.getOperand(0), AM))
2812 Src.getOperand(0), AM, *Subtarget))
2820 return matchAddressBase(
N, AM);
2825bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2827 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2829 if (!AM.IndexReg.getNode()) {
2840 AM.BaseType = X86ISelAddressMode::RegBase;
2845bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2846 X86ISelAddressMode &AM,
2850 dbgs() <<
"MatchVectorAddress: ";
2855 return matchAddressBase(
N, AM);
2858 switch (
N.getOpcode()) {
2860 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2861 if (!foldOffsetIntoAddress(Val, AM))
2866 if (!matchWrapper(
N, AM))
2874 X86ISelAddressMode Backup = AM;
2875 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2876 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2882 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2884 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2889 N = Handle.getValue();
2894 return matchAddressBase(
N, AM);
2900bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2901 return matchVectorAddressRecursively(
N, AM, 0);
2909 X86ISelAddressMode AM;
2915 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2917 AM.IndexReg = IndexOp;
2921 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2923 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2925 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2931 if (matchVectorAddress(BasePtr, AM))
2934 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2948 X86ISelAddressMode AM;
2960 unsigned AddrSpace =
2961 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2963 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2965 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2967 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2972 MVT VT =
N.getSimpleValueType();
2974 if (matchAddress(
N, AM))
2977 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2993 N =
N.getOperand(0);
3008 const GlobalValue *GV = cast<GlobalAddressSDNode>(
N)->getGlobal();
3010 return CR->getUnsignedMax().ult(1ull << 32);
3012 return !
TM.isLargeGlobalValue(GV);
3021 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
3024 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
3025 if (RN &&
RN->getReg() == 0)
3026 Base = CurDAG->getRegister(0, MVT::i64);
3027 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
3031 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3035 RN = dyn_cast<RegisterSDNode>(
Index);
3036 if (RN &&
RN->getReg() == 0)
3037 Index = CurDAG->getRegister(0, MVT::i64);
3040 "Expect to be extending 32-bit registers for use in LEA");
3043 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3052bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
3056 X86ISelAddressMode AM;
3060 MVT VT =
N.getSimpleValueType();
3065 SDValue T = CurDAG->getRegister(0, MVT::i32);
3067 if (matchAddress(
N, AM))
3072 unsigned Complexity = 0;
3073 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3075 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3078 if (AM.IndexReg.getNode())
3091 if (AM.hasSymbolicDisplacement()) {
3093 if (Subtarget->is64Bit())
3103 auto isMathWithFlags = [](
SDValue V) {
3104 switch (
V.getOpcode()) {
3125 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3133 if (Complexity <= 2)
3136 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3147 X86ISelAddressMode AM;
3148 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N)) {
3149 AM.GV = GA->getGlobal();
3150 AM.Disp += GA->getOffset();
3151 AM.SymbolFlags = GA->getTargetFlags();
3153 auto *SA = cast<ExternalSymbolSDNode>(
N);
3154 AM.ES = SA->getSymbol();
3155 AM.SymbolFlags = SA->getTargetFlags();
3158 if (Subtarget->is32Bit()) {
3160 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3163 MVT VT =
N.getSimpleValueType();
3164 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3172 EVT VT =
N.getValueType();
3173 bool WasTruncated =
false;
3175 WasTruncated =
true;
3176 N =
N.getOperand(0);
3185 unsigned Opc =
N.getOperand(0)->getOpcode();
3187 Op =
N.getOperand(0);
3190 return !WasTruncated;
3194 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3195 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3196 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3200 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3201 GA->getOffset(), GA->getTargetFlags());
3209 assert(Root &&
P &&
"Unknown root/parent nodes");
3211 !IsProfitableToFold(
N,
P, Root) ||
3212 !IsLegalToFold(
N,
P, Root, OptLevel))
3215 return selectAddr(
N.getNode(),
3216 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3223 assert(Root &&
P &&
"Unknown root/parent nodes");
3225 !IsProfitableToFold(
N,
P, Root) ||
3226 !IsLegalToFold(
N,
P, Root, OptLevel))
3229 return selectAddr(
N.getNode(),
3230 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3236SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3237 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3239 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3242bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3244 N =
N->getOperand(0).getNode();
3248 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3252 auto *GV = GA->getGlobal();
3255 return CR->getSignedMin().sge(-1ull << Width) &&
3256 CR->getSignedMax().slt(1ull << Width);
3261 return Width == 32 && !
TM.isLargeGlobalValue(GV);
3265 assert(
N->isMachineOpcode() &&
"Unexpected node");
3266 unsigned Opc =
N->getMachineOpcode();
3267 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3272 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3277bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3282 if (UI.getUse().getResNo() !=
Flags.getResNo())
3286 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3290 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3292 if (FlagUI.getUse().getResNo() != 1)
continue;
3294 if (!FlagUI->isMachineOpcode())
return false;
3313bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3318 if (UI.getUse().getResNo() !=
Flags.getResNo())
3322 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3326 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3328 if (FlagUI.getUse().getResNo() != 1)
continue;
3330 if (!FlagUI->isMachineOpcode())
return false;
3369 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3374 if (UI.getUse().getResNo() !=
Flags.getResNo())
3377 unsigned UIOpc = UI->getOpcode();
3381 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3385 FlagUI != FlagUE; ++FlagUI) {
3387 if (FlagUI.getUse().getResNo() != 1)
3390 if (!FlagUI->isMachineOpcode())
3431 if (StoredVal.
getResNo() != 0)
return false;
3445 LoadNode = cast<LoadSDNode>(Load);
3448 if (!Load.hasOneUse())
3456 bool FoundLoad =
false;
3460 const unsigned int Max = 1024;
3502 if (Chain == Load.getValue(1)) {
3508 if (
Op == Load.getValue(1)) {
3524 if (
Op.getNode() != LoadNode)
3557bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3558 auto *StoreNode = cast<StoreSDNode>(
Node);
3565 EVT MemVT = StoreNode->getMemoryVT();
3566 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3570 bool IsCommutable =
false;
3571 bool IsNegate =
false;
3585 IsCommutable =
true;
3589 unsigned LoadOpNo = IsNegate ? 1 : 0;
3593 LoadNode, InputChain)) {
3600 LoadNode, InputChain))
3609 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3630 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3640 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3644 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3647 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3648 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3661 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3664 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3667 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3670 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3673 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3676 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3679 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3681 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3687 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3690 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3693 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3696 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3699 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3702 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3705 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3708 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3715 unsigned NewOpc = SelectRegOpcode(Opc);
3720 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3721 int64_t OperandV = OperandC->getSExtValue();
3727 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3728 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3729 isInt<32>(-OperandV))) &&
3730 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3731 OperandV = -OperandV;
3735 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3736 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3737 NewOpc = SelectImmOpcode(Opc);
3743 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3747 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3748 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3752 Segment, Operand, InputChain};
3753 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3764 CurDAG->setNodeMemRefs(Result, MemOps);
3770 CurDAG->RemoveDeadNode(
Node);
3781bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3785 "Should be either an and-mask, or right-shift after clearing high bits.");
3788 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3791 MVT NVT =
Node->getSimpleValueType(0);
3794 if (NVT != MVT::i32 && NVT != MVT::i64)
3802 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3803 auto checkUses = [AllowExtraUsesByDefault](
3805 std::optional<bool> AllowExtraUses) {
3806 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3807 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3809 auto checkOneUse = [checkUses](
SDValue Op,
3810 std::optional<bool> AllowExtraUses =
3812 return checkUses(
Op, 1, AllowExtraUses);
3814 auto checkTwoUse = [checkUses](
SDValue Op,
3815 std::optional<bool> AllowExtraUses =
3817 return checkUses(
Op, 2, AllowExtraUses);
3820 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3822 assert(
V.getSimpleValueType() == MVT::i32 &&
3823 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3824 "Expected i64 -> i32 truncation");
3825 V =
V.getOperand(0);
3831 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3834 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3840 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3845 NBits =
M0->getOperand(1);
3846 NegateNBits =
false;
3850 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3851 V = peekThroughOneUseTruncation(V);
3852 return CurDAG->MaskedValueIsAllOnes(
3858 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3861 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3864 if (!isAllOnes(
Mask->getOperand(1)))
3867 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3871 if (!isAllOnes(
M0->getOperand(0)))
3873 NBits =
M0->getOperand(1);
3874 NegateNBits =
false;
3880 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3881 unsigned Bitwidth) {
3886 NBits = NBits.getOperand(0);
3891 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3892 if (!V0 || V0->getZExtValue() != Bitwidth)
3894 NBits = NBits.getOperand(1);
3895 NegateNBits =
false;
3901 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3904 Mask = peekThroughOneUseTruncation(Mask);
3905 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3907 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3914 if (!checkOneUse(
M1))
3916 canonicalizeShiftAmt(
M1, Bitwidth);
3921 return !NegateNBits;
3929 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3930 AllowExtraUsesByDefault, &NegateNBits,
3943 canonicalizeShiftAmt(N1, Bitwidth);
3947 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3948 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3954 auto matchLowBitMask = [matchPatternA, matchPatternB,
3956 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3960 X =
Node->getOperand(0);
3963 if (matchLowBitMask(Mask)) {
3967 if (!matchLowBitMask(Mask))
3971 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3972 }
else if (!matchPatternD(
Node))
3977 if (NegateNBits && !Subtarget->hasBMI2())
3989 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3992 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3994 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3995 MVT::i32, ImplDef, NBits, SRIdxVal),
4005 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
4009 if (Subtarget->hasBMI2()) {
4011 if (NVT != MVT::i32) {
4019 SelectCode(Extract.
getNode());
4028 SDValue RealX = peekThroughOneUseTruncation(
X);
4034 MVT XVT =
X.getSimpleValueType();
4044 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
4052 SDValue ShiftAmt =
X.getOperand(1);
4053 X =
X.getOperand(0);
4056 "Expected shift amount to be i8");
4060 SDValue OrigShiftAmt = ShiftAmt;
4065 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
4070 if (XVT != MVT::i32) {
4085 SelectCode(Extract.
getNode());
4092 MVT NVT =
Node->getSimpleValueType(0);
4105 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4106 if (!PreferBEXTR && !Subtarget->hasBMI2())
4118 if (NVT != MVT::i32 && NVT != MVT::i64)
4122 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4123 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4124 if (!MaskCst || !ShiftCst)
4132 uint64_t Shift = ShiftCst->getZExtValue();
4137 if (Shift == 8 && MaskSize == 8)
4148 if (!PreferBEXTR && MaskSize <= 32)
4152 unsigned ROpc, MOpc;
4154#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4156 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4160 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4165 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4166 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4172 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4173 if (Subtarget->hasTBM()) {
4174 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4175 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4177 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4183 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4184 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4190 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4191 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4193 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4194 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4195 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4199 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4201 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4206 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4210 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4217MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4218 bool MayFoldLoad,
const SDLoc &dl,
4223 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4224 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4227 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4228 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4229 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4231 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4232 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4236 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4241 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4242 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4249MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4250 bool MayFoldLoad,
const SDLoc &dl,
4256 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4257 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4260 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4261 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4262 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4264 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4265 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4270 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4275 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4276 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4281bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4282 EVT VT =
N->getValueType(0);
4289 unsigned Size = VT == MVT::i64 ? 64 : 32;
4291 SDValue OrigShiftAmt =
N->getOperand(1);
4292 SDValue ShiftAmt = OrigShiftAmt;
4307 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4308 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4311 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4315 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4316 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4320 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4329 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4330 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4336 Add0C->getZExtValue() != 0) {
4339 if (Add0C->getZExtValue() %
Size == 0)
4342 Add0C->getZExtValue() % 32 == 0) {
4350 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4354 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4376 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4383 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4384 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4388 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4390 if (UpdatedNode !=
N) {
4393 ReplaceNode(
N, UpdatedNode);
4400 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4408bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4409 MVT NVT =
N->getSimpleValueType(0);
4410 unsigned Opcode =
N->getOpcode();
4418 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4422 int64_t Val = Cst->getSExtValue();
4427 bool FoundAnyExtend =
false;
4431 FoundAnyExtend =
true;
4439 if (NVT != MVT::i32 && NVT != MVT::i64)
4442 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4446 uint64_t ShAmt = ShlCst->getZExtValue();
4450 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4451 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4456 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4460 ShiftedVal = (
uint64_t)Val >> ShAmt;
4461 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4464 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4467 ShiftedVal = Val >> ShAmt;
4468 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4469 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4473 ShiftedVal = (
uint64_t)Val >> ShAmt;
4474 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4481 if (!CanShrinkImmediate(ShiftedVal))
4491 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4497 NeededMask &= ~Cst->getAPIntValue();
4499 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4504 if (FoundAnyExtend) {
4510 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4512 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4521bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4525 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4526 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4528 auto tryFoldLoadOrBCast =
4531 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4537 L =
L.getOperand(0);
4544 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4545 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4549 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4552 bool FoldedLoad =
false;
4553 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4554 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4556 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4561 uint8_t OldImm =
Imm;
4562 Imm = OldImm & 0xa5;
4563 if (OldImm & 0x02)
Imm |= 0x10;
4564 if (OldImm & 0x10)
Imm |= 0x02;
4565 if (OldImm & 0x08)
Imm |= 0x40;
4566 if (OldImm & 0x40)
Imm |= 0x08;
4567 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4572 uint8_t OldImm =
Imm;
4573 Imm = OldImm & 0x99;
4574 if (OldImm & 0x02)
Imm |= 0x04;
4575 if (OldImm & 0x04)
Imm |= 0x02;
4576 if (OldImm & 0x20)
Imm |= 0x40;
4577 if (OldImm & 0x40)
Imm |= 0x20;
4582 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4588 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4592 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4593 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4594 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4596 bool UseD = EltSize == 32;
4598 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4600 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4602 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4608 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4610 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4612 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4617 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4618 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4621 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4623 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4628 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4630 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4632 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4636 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4640 CurDAG->RemoveDeadNode(Root);
4646bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4647 MVT NVT =
N->getSimpleValueType(0);
4650 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4661 auto getFoldableLogicOp = [](
SDValue Op) {
4664 Op =
Op.getOperand(0);
4666 if (!
Op.hasOneUse())
4669 unsigned Opc =
Op.getOpcode();
4678 if ((FoldableOp = getFoldableLogicOp(N1))) {
4680 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4693 uint8_t TernlogMagicA = 0xf0;
4694 uint8_t TernlogMagicB = 0xcc;
4695 uint8_t TernlogMagicC = 0xaa;
4704 Parent =
Op.getNode();
4705 Op =
Op.getOperand(0);
4709 PeekThroughNot(
A, ParentA, TernlogMagicA);
4710 PeekThroughNot(
B, ParentB, TernlogMagicB);
4711 PeekThroughNot(
C, ParentC, TernlogMagicC);
4716 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4717 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4718 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4722 switch (
N->getOpcode()) {
4726 Imm &= ~TernlogMagicA;
4728 Imm = ~(
Imm) & TernlogMagicA;
4735 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4745bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4748 MVT VT =
And->getSimpleValueType(0);
4749 if (VT != MVT::i32 && VT != MVT::i64)
4752 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4761 APInt MaskVal = And1C->getAPIntValue();
4763 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4767 if (VT == MVT::i64 && MaskLZ >= 32) {
4769 MaskVal = MaskVal.
trunc(32);
4774 APInt NegMaskVal = MaskVal | HighZeros;
4783 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4784 NegMaskVal = NegMaskVal.
zext(64);
4785 HighZeros = HighZeros.
zext(64);
4790 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4810 bool FoldedBCast,
bool Masked) {
4811#define VPTESTM_CASE(VT, SUFFIX) \
4814 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4815 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4818#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4819default: llvm_unreachable("Unexpected VT!"); \
4820VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4821VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4822VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4823VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4824VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4825VPTESTM_CASE(v8i64, QZ##SUFFIX)
4827#define VPTESTM_FULL_CASES(SUFFIX) \
4828VPTESTM_BROADCAST_CASES(SUFFIX) \
4829VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4830VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4831VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4832VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4833VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4834VPTESTM_CASE(v32i16, WZ##SUFFIX)
4852#undef VPTESTM_FULL_CASES
4853#undef VPTESTM_BROADCAST_CASES
4859bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4861 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4911 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4916 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4922 L =
L.getOperand(0);
4928 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4929 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4932 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4936 bool CanFoldLoads = Src0 != Src1;
4938 bool FoldedLoad =
false;
4939 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4941 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4945 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4954 bool IsMasked = InMask.
getNode() !=
nullptr;
4967 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4969 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4972 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4977 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4978 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4979 dl, MaskVT, InMask, RC), 0);
4984 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4989 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4992 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4994 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4996 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4998 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5004 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
5007 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
5009 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
5015 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
5016 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
5017 dl, ResVT,
SDValue(CNode, 0), RC);
5021 CurDAG->RemoveDeadNode(Root);
5027bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
5030 MVT NVT =
N->getSimpleValueType(0);
5033 if (!NVT.
isVector() || !Subtarget->hasAVX512())
5067 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5076 MVT NVT =
Node->getSimpleValueType(0);
5077 unsigned Opcode =
Node->getOpcode();
5080 if (
Node->isMachineOpcode()) {
5082 Node->setNodeId(-1);
5089 unsigned IntNo =
Node->getConstantOperandVal(1);
5092 case Intrinsic::x86_encodekey128:
5093 case Intrinsic::x86_encodekey256: {
5094 if (!Subtarget->hasKL())
5100 case Intrinsic::x86_encodekey128:
5101 Opcode = X86::ENCODEKEY128;
5103 case Intrinsic::x86_encodekey256:
5104 Opcode = X86::ENCODEKEY256;
5109 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
5111 if (Opcode == X86::ENCODEKEY256)
5112 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
5116 Opcode, dl,
Node->getVTList(),
5117 {Node->getOperand(2), Chain, Chain.getValue(1)});
5118 ReplaceNode(
Node, Res);
5121 case Intrinsic::x86_tileloadd64_internal:
5122 case Intrinsic::x86_tileloaddt164_internal: {
5123 if (!Subtarget->hasAMXTILE())
5128 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
5130 : X86::PTILELOADDT1V;
5133 SDValue Scale = getI8Imm(1, dl);
5135 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5136 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5140 Node->getOperand(3),
5147 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5148 ReplaceNode(
Node, CNode);
5155 unsigned IntNo =
Node->getConstantOperandVal(1);
5158 case Intrinsic::x86_sse3_monitor:
5159 case Intrinsic::x86_monitorx:
5160 case Intrinsic::x86_clzero: {
5161 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
5166 case Intrinsic::x86_sse3_monitor:
5167 if (!Subtarget->hasSSE3())
5169 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5171 case Intrinsic::x86_monitorx:
5172 if (!Subtarget->hasMWAITX())
5174 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5176 case Intrinsic::x86_clzero:
5177 if (!Subtarget->hasCLZERO())
5179 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5184 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5185 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
5189 if (IntNo == Intrinsic::x86_sse3_monitor ||
5190 IntNo == Intrinsic::x86_monitorx) {
5192 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
5195 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
5200 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5202 ReplaceNode(
Node, CNode);
5208 case Intrinsic::x86_tilestored64_internal: {
5212 unsigned Opc = X86::PTILESTOREDV;
5215 SDValue Scale = getI8Imm(1, dl);
5217 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5218 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5222 Node->getOperand(3),
5228 Node->getOperand(6),
5230 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5231 ReplaceNode(
Node, CNode);
5234 case Intrinsic::x86_tileloadd64:
5235 case Intrinsic::x86_tileloaddt164:
5236 case Intrinsic::x86_tilestored64: {
5237 if (!Subtarget->hasAMXTILE())
5245 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD;
break;
5246 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1;
break;
5247 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED;
break;
5250 unsigned TIndex =
Node->getConstantOperandVal(2);
5251 SDValue TReg = getI8Imm(TIndex, dl);
5253 SDValue Scale = getI8Imm(1, dl);
5255 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5256 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5259 if (Opc == X86::PTILESTORED) {
5261 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5264 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5266 ReplaceNode(
Node, CNode);
5274 if (Subtarget->isTargetNaCl())
5278 if (Subtarget->isTarget64BitILP32()) {
5283 assert(
Target.getValueType() == MVT::i32 &&
"Unexpected VT!");
5284 SDValue ZextTarget = CurDAG->getZExtOrTrunc(
Target, dl, MVT::i64);
5285 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5286 Node->getOperand(0), ZextTarget);
5288 SelectCode(ZextTarget.
getNode());
5295 ReplaceNode(
Node, getGlobalBaseReg());
5303 CurDAG->RemoveDeadNode(
Node);
5309 if (matchBitExtract(
Node))
5314 if (tryShiftAmountMod(
Node))
5319 uint8_t
Imm =
Node->getConstantOperandVal(3);
5321 Node->getOperand(1),
Node->getOperand(2), Imm))
5327 if (tryVPTERNLOG(
Node))
5337 tryVPTESTM(
Node, N0, N1))
5340 tryVPTESTM(
Node, N1, N0))
5346 CurDAG->RemoveDeadNode(
Node);
5349 if (matchBitExtract(
Node))
5357 if (tryShrinkShlLogicImm(
Node))
5361 if (tryVPTERNLOG(
Node))
5376 if (!CurDAG->shouldOptForSize())
5380 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5386 auto *Cst = dyn_cast<ConstantSDNode>(N1);
5390 int64_t Val = Cst->getSExtValue();
5394 if (!isInt<8>(Val) && !isInt<32>(Val))
5398 if (Opcode ==
ISD::ADD && (Val == 1 || Val == -1))
5402 if (!shouldAvoidImmediateInstFormsForSize(N1.
getNode()))
5406 unsigned ROpc, MOpc;
5515 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5516 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5518 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5519 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5523 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()});
5525 CurDAG->RemoveDeadNode(
Node);
5530 CurDAG->SelectNodeTo(
Node, ROpc, NVT, MVT::i32, N0, N1);
5543 unsigned LoReg, ROpc, MOpc;
5548 ROpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5549 MOpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5568 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5569 bool FoldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5572 FoldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5577 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5586 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5588 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5592 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5597 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5603 VTs = CurDAG->getVTList(NVT, MVT::i32);
5605 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5607 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5612 CurDAG->RemoveDeadNode(
Node);
5622 unsigned LoReg, HiReg;
5624 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5629 Opc = UseMULXHi ? X86::MULX32Hrr
5631 : IsSigned ?
X86::IMUL32r
5633 MOpc = UseMULXHi ? X86::MULX32Hrm
5635 : IsSigned ?
X86::IMUL32m
5637 LoReg = UseMULX ? X86::EDX : X86::EAX;
5641 Opc = UseMULXHi ? X86::MULX64Hrr
5643 : IsSigned ?
X86::IMUL64r
5645 MOpc = UseMULXHi ? X86::MULX64Hrm
5647 : IsSigned ?
X86::IMUL64m
5649 LoReg = UseMULX ? X86::RDX : X86::RAX;
5654 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5655 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5658 foldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5663 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5672 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5673 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5676 }
else if (UseMULX) {
5677 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5678 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5683 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5684 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5690 ReplaceUses(N1.
getValue(1), Chain);
5692 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5694 SDValue Ops[] = { N1, InGlue };
5696 SDVTList VTs = CurDAG->getVTList(NVT);
5697 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5699 }
else if (UseMULX) {
5700 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5701 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5705 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5706 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5714 assert(LoReg &&
"Register for low half is not defined!");
5715 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5726 assert(HiReg &&
"Register for high half is not defined!");
5727 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5736 CurDAG->RemoveDeadNode(
Node);
5745 unsigned ROpc, MOpc;
5750 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m;
break;
5751 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m;
break;
5752 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m;
break;
5753 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m;
break;
5758 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
5759 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
5760 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
5761 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
5765 unsigned LoReg, HiReg, ClrReg;
5766 unsigned SExtOpcode;
5770 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5774 LoReg = X86::AX; HiReg = X86::DX;
5776 SExtOpcode = X86::CWD;
5779 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5780 SExtOpcode = X86::CDQ;
5783 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5784 SExtOpcode = X86::CQO;
5788 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5789 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5790 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5793 if (NVT == MVT::i8) {
5796 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5798 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5800 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5802 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5804 ReplaceUses(N0.
getValue(1), Chain);
5806 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
5808 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5810 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5811 Chain = CurDAG->getEntryNode();
5813 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX,
SDValue(Move, 0),
5818 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5819 LoReg, N0,
SDValue()).getValue(1);
5823 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5826 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5828 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
5832 SDValue(CurDAG->getMachineNode(
5833 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5834 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5842 SDValue(CurDAG->getMachineNode(
5843 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5844 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5845 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5853 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5854 ClrNode, InGlue).getValue(1);
5862 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5867 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5870 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5880 if (HiReg == X86::AH && !
SDValue(
Node, 1).use_empty()) {
5881 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5882 unsigned AHExtOpcode =
5883 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5885 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
5886 MVT::Glue, AHCopy, InGlue);
5891 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
5899 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5900 LoReg, NVT, InGlue);
5901 InGlue =
Result.getValue(2);
5908 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5909 HiReg, NVT, InGlue);
5910 InGlue =
Result.getValue(2);
5915 CurDAG->RemoveDeadNode(
Node);
5924 SDValue N0 =
Node->getOperand(IsStrictCmp ? 1 : 0);
5925 SDValue N1 =
Node->getOperand(IsStrictCmp ? 2 : 1);
5931 if (Subtarget->canUseCMOV())
5940 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
5943 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
5946 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
5951 IsStrictCmp ?
Node->getOperand(0) : CurDAG->getEntryNode();
5954 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5955 Chain =
SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5958 Glue =
SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
5963 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
5967 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
5971 assert(Subtarget->canUseLAHFSAHF() &&
5972 "Target doesn't support SAHF or FCOMI?");
5973 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract,
SDValue());
5976 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.
getValue(1)), 0);
5982 CurDAG->RemoveDeadNode(
Node);
6003 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
6006 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
6008 CurDAG->RemoveDeadNode(
Node);
6022 auto *MaskC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
6040 unsigned TestOpcode;
6048 if (LeadingZeros == 0 && SavesBytes) {
6053 ShiftAmt = TrailingZeros;
6055 TestOpcode = X86::TEST64rr;
6056 }
else if (TrailingZeros == 0 && SavesBytes) {
6061 ShiftAmt = LeadingZeros;
6063 TestOpcode = X86::TEST64rr;
6064 }
else if (MaskC->hasOneUse() && !isInt<32>(Mask)) {
6067 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6068 if (PopCount == 8) {
6070 ShiftAmt = TrailingZeros;
6071 SubRegIdx = X86::sub_8bit;
6073 TestOpcode = X86::TEST8rr;
6074 }
else if (PopCount == 16) {
6076 ShiftAmt = TrailingZeros;
6077 SubRegIdx = X86::sub_16bit;
6078 SubRegVT = MVT::i16;
6079 TestOpcode = X86::TEST16rr;
6080 }
else if (PopCount == 32) {
6082 ShiftAmt = TrailingZeros;
6083 SubRegIdx = X86::sub_32bit;
6084 SubRegVT = MVT::i32;
6085 TestOpcode = X86::TEST32rr;
6089 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6091 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6094 if (SubRegIdx != 0) {
6096 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6099 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6107 unsigned ROpc, MOpc;
6114 if (isUInt<8>(Mask) &&
6115 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6119 SubRegOp = X86::sub_8bit;
6120 ROpc = X86::TEST8ri;
6121 MOpc = X86::TEST8mi;
6122 }
else if (OptForMinSize && isUInt<16>(Mask) &&
6123 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6130 SubRegOp = X86::sub_16bit;
6131 ROpc = X86::TEST16ri;
6132 MOpc = X86::TEST16mi;
6133 }
else if (isUInt<32>(Mask) && N0.
getValueType() != MVT::i16 &&
6134 ((!(Mask & 0x80000000) &&
6137 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6138 CmpVT == MVT::i32 ||
6146 SubRegOp = X86::sub_32bit;
6147 ROpc = X86::TEST32ri;
6148 MOpc = X86::TEST32mi;
6154 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6159 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6160 if (tryFoldLoad(
Node, N0.
getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6162 if (!LoadN->isSimple()) {
6164 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6165 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6166 (MOpc == X86::TEST32mi && NumVolBits != 32))
6170 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
6171 Reg.getOperand(0) };
6172 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6174 ReplaceUses(
Reg.getValue(1),
SDValue(NewNode, 1));
6176 CurDAG->setNodeMemRefs(NewNode,
6177 {cast<LoadSDNode>(Reg)->getMemOperand()});
6181 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6183 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6186 ReplaceNode(
Node, NewNode);
6192 if (!Subtarget->hasSSE42())
6198 bool MayFoldLoad = !NeedIndex || !NeedMask;
6203 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6205 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6206 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node);
6209 if (NeedIndex || !NeedMask) {
6211 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6213 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6214 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node);
6220 CurDAG->RemoveDeadNode(
Node);
6224 if (!Subtarget->hasSSE42())
6228 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6229 Node->getOperand(1),
6231 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6232 Node->getOperand(3), InGlue).getValue(1);
6237 bool MayFoldLoad = !NeedIndex || !NeedMask;
6242 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6244 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6246 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node, InGlue);
6249 if (NeedIndex || !NeedMask) {
6251 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6253 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6254 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node, InGlue);
6259 CurDAG->RemoveDeadNode(
Node);
6271 if (foldLoadStoreIntoMemOperand(
Node))
6276 MVT VT =
Node->getSimpleValueType(0);
6278 if (Subtarget->hasSBBDepBreaking()) {
6283 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6288 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6289 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6291 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.
getValue(1)),
6300 if (VT == MVT::i8 || VT == MVT::i16) {
6301 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6302 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6306 CurDAG->RemoveDeadNode(
Node);
6320 MVT VT =
Node->getSimpleValueType(0);
6321 if (VT == MVT::i8 || VT == MVT::i16) {
6322 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6323 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6328 CurDAG->RemoveDeadNode(
Node);
6334 auto *Mgt = cast<X86MaskedGatherSDNode>(
Node);
6335 SDValue IndexOp = Mgt->getIndex();
6338 MVT ValueVT =
Node->getSimpleValueType(0);
6339 MVT MaskVT =
Mask.getSimpleValueType();
6356 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6357 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6358 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6359 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6360 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6361 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6362 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6363 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6364 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6365 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6366 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6367 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6368 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6369 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6370 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6371 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6372 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6373 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6374 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6375 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6376 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6377 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6378 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6379 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6381 assert(
EVT(MaskVT) ==
EVT(ValueVT).changeVectorElementTypeToInteger() &&
6382 "Unexpected mask VT!");
6383 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6384 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6385 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6386 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6387 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6388 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6389 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6390 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6391 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6392 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6393 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6394 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6395 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6396 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6397 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6398 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6405 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6409 SDValue PassThru = Mgt->getPassThru();
6410 SDValue Chain = Mgt->getChain();
6412 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6417 Index, Disp, Segment, Chain};
6418 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6421 Disp, Segment,
Mask, Chain};
6422 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6424 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6427 CurDAG->RemoveDeadNode(
Node);
6431 auto *Sc = cast<X86MaskedScatterSDNode>(
Node);
6433 SDValue IndexOp = Sc->getIndex();
6435 MVT ValueVT =
Value.getSimpleValueType();
6450 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6451 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6452 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6453 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6454 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6455 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6456 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6457 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6458 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6459 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6460 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6461 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6462 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6463 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6464 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6465 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6466 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6467 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6468 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6469 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6470 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6471 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6472 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6473 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6478 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6483 SDValue Chain = Sc->getChain();
6485 SDVTList VTs = CurDAG->getVTList(
Mask.getValueType(), MVT::Other);
6489 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6491 CurDAG->RemoveDeadNode(
Node);
6497 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6499 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6501 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6503 CurDAG->RemoveDeadNode(
Node);
6509 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6511 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6514 Ops[0] = CallIdValue;
6518 TargetOpcode::PREALLOCATED_ARG, dl,
6519 CurDAG->getVTList(TLI->
getPointerTy(CurDAG->getDataLayout()),
6524 CurDAG->RemoveDeadNode(
Node);
6531 if (!Subtarget->hasWIDEKL())
6535 switch (
Node->getOpcode()) {
6539 Opcode = X86::AESENCWIDE128KL;
6542 Opcode = X86::AESDECWIDE128KL;
6545 Opcode = X86::AESENCWIDE256KL;
6548 Opcode = X86::AESDECWIDE256KL;
6559 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(2),
6561 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(3),
6563 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2,
Node->getOperand(4),
6565 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3,
Node->getOperand(5),
6567 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4,
Node->getOperand(6),
6569 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5,
Node->getOperand(7),
6571 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6,
Node->getOperand(8),
6573 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7,
Node->getOperand(9),
6577 Opcode, dl,
Node->getVTList(),
6578 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6579 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(
Node)->getMemOperand());
6580 ReplaceNode(
Node, Res);
6588bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6590 std::vector<SDValue> &OutOps) {
6591 SDValue Op0, Op1, Op2, Op3, Op4;
6592 switch (ConstraintID) {
6595 case InlineAsm::ConstraintCode::o:
6596 case InlineAsm::ConstraintCode::v:
6597 case InlineAsm::ConstraintCode::m:
6598 case InlineAsm::ConstraintCode::X:
6599 case InlineAsm::ConstraintCode::p:
6600 if (!selectAddr(
nullptr,
Op, Op0, Op1, Op2, Op3, Op4))
6605 OutOps.push_back(Op0);
6606 OutOps.push_back(Op1);
6607 OutOps.push_back(Op2);
6608 OutOps.push_back(Op3);
6609 OutOps.push_back(Op4);
6615 std::make_unique<X86DAGToDAGISel>(
TM,
TM.getOptLevel())) {}
6621 return new X86DAGToDAGISelLegacy(
TM, OptLevel);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
amdgpu AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Module.h This file contains the declarations for the Module class.
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to mod...
#define GET_EGPR_IF_ENABLED(OPC)
static bool needBWI(MVT VT)
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool mayUseCarryFlag(X86::CondCode CC)
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
cl::opt< bool > IndirectBranchTracking
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
static bool isDispSafeForFrameIndex(int64_t Val)
static bool isEndbrImm64(uint64_t Imm)
#define GET_ND_IF_ENABLED(OPC)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
The address of a basic block.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Module * getParent()
Get the module that this global value is contained inside of...
std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
static int getUninvalidatedNodeId(SDNode *N)
virtual void emitFunctionEntryCode()
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static Type * getVoidTy(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void dump() const
Support for debugging, callable in GDB: V->dump()
X86ISelDAGToDAGPass(X86TargetMachine &TM)
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
void setAMXProgModel(AMXProgModelEnum Model)
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
CodeGenOptLevel
Code generation optimization level.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.