22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40STATISTIC(NumLoadMoved,
"Number of loads moved below TokenFactor");
43 cl::desc(
"Enable setting constant bits to reduce size of mask immediates"),
47 "x86-promote-anyext-load",
cl::init(
true),
59 struct X86ISelAddressMode {
67 int Base_FrameIndex = 0;
76 const char *ES =
nullptr;
81 bool NegateIndex =
false;
83 X86ISelAddressMode() =
default;
85 bool hasSymbolicDisplacement()
const {
86 return GV !=
nullptr ||
CP !=
nullptr || ES !=
nullptr ||
87 MCSym !=
nullptr ||
JT != -1 || BlockAddr !=
nullptr;
90 bool hasBaseOrIndexReg()
const {
97 if (
BaseType != RegBase)
return false;
99 dyn_cast_or_null<RegisterSDNode>(Base_Reg.
getNode()))
100 return RegNode->getReg() == X86::RIP;
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 dbgs() <<
"X86ISelAddressMode " <<
this <<
'\n';
112 dbgs() <<
"Base_Reg ";
118 dbgs() <<
" Base.FrameIndex " << Base_FrameIndex <<
'\n';
119 dbgs() <<
" Scale " << Scale <<
'\n'
127 dbgs() <<
" Disp " << Disp <<
'\n'
149 dbgs() <<
" JT" <<
JT <<
" Align" << Alignment.
value() <<
'\n';
169 bool IndirectTlsSegRefs;
174 X86DAGToDAGISel() =
delete;
178 OptForMinSize(
false), IndirectTlsSegRefs(
false) {}
184 "indirect-tls-seg-refs");
189 "OptForMinSize implies OptForSize");
203#include "X86GenDAGISel.inc"
208 bool foldOffsetIntoAddress(
uint64_t Offset, X86ISelAddressMode &AM);
209 bool matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
210 bool AllowSegmentRegForX32 =
false);
211 bool matchWrapper(
SDValue N, X86ISelAddressMode &AM);
212 bool matchAddress(
SDValue N, X86ISelAddressMode &AM);
213 bool matchVectorAddress(
SDValue N, X86ISelAddressMode &AM);
214 bool matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
unsigned Depth);
217 bool matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
219 bool matchVectorAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
221 bool matchAddressBase(
SDValue N, X86ISelAddressMode &AM);
250 return tryFoldLoad(
P,
P,
N,
Base, Scale,
Index, Disp, Segment);
258 bool isProfitableToFormMaskedOp(
SDNode *
N)
const;
263 std::vector<SDValue> &OutOps)
override;
265 void emitSpecialCodeForMain();
267 inline void getAddressOperands(X86ISelAddressMode &AM,
const SDLoc &
DL,
271 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
272 Base = CurDAG->getTargetFrameIndex(
273 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
274 else if (AM.Base_Reg.getNode())
277 Base = CurDAG->getRegister(0, VT);
279 Scale = getI8Imm(AM.Scale,
DL);
281#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
283 if (AM.NegateIndex) {
291 if (AM.IndexReg.getNode())
294 Index = CurDAG->getRegister(0, VT);
299 Disp = CurDAG->getTargetGlobalAddress(AM.GV,
SDLoc(),
303 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
304 AM.Disp, AM.SymbolFlags);
306 assert(!AM.Disp &&
"Non-zero displacement is ignored with ES.");
307 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
308 }
else if (AM.MCSym) {
309 assert(!AM.Disp &&
"Non-zero displacement is ignored with MCSym.");
310 assert(AM.SymbolFlags == 0 &&
"oo");
311 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
312 }
else if (AM.JT != -1) {
313 assert(!AM.Disp &&
"Non-zero displacement is ignored with JT.");
314 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
315 }
else if (AM.BlockAddr)
316 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
319 Disp = CurDAG->getTargetConstant(AM.Disp,
DL, MVT::i32);
321 if (AM.Segment.getNode())
322 Segment = AM.Segment;
324 Segment = CurDAG->getRegister(0, MVT::i16);
333 bool shouldAvoidImmediateInstFormsForSize(
SDNode *
N)
const {
339 if (!CurDAG->shouldOptForSize())
349 if (
User->isMachineOpcode()) {
372 auto *
C = dyn_cast<ConstantSDNode>(
N);
373 if (
C && isInt<8>(
C->getSExtValue()))
393 (RegNode = dyn_cast_or_null<RegisterSDNode>(
395 if ((RegNode->
getReg() == X86::ESP) ||
396 (RegNode->
getReg() == X86::RSP))
405 return (UseCount > 1);
410 return CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
415 return CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
420 return CurDAG->getTargetConstant(Imm,
DL, MVT::i64);
425 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
427 MVT VecVT =
N->getOperand(0).getSimpleValueType();
433 assert((VecWidth == 128 || VecWidth == 256) &&
"Unexpected vector width");
435 MVT VecVT =
N->getSimpleValueType(0);
439 SDValue getPermuteVINSERTCommutedImmediate(
SDNode *
N,
unsigned VecWidth,
441 assert(VecWidth == 128 &&
"Unexpected vector width");
443 MVT VecVT =
N->getSimpleValueType(0);
445 assert((InsertIdx == 0 || InsertIdx == 1) &&
"Bad insertf128 index");
448 return getI8Imm(InsertIdx ? 0x02 : 0x30,
DL);
453 MVT VT =
N->getSimpleValueType(0);
456 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
458 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
459 if (VT == MVT::i64) {
461 CurDAG->getMachineNode(
462 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
463 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
464 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
469 unsigned Opcode =
N->getOpcode();
471 "Unexpected opcode for SBB materialization");
472 unsigned FlagOpIndex = Opcode ==
X86ISD::SBB ? 2 : 1;
474 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
475 N->getOperand(FlagOpIndex),
SDValue());
479 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
480 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
481 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
483 CurDAG->getMachineNode(Opc, dl, VTs,
484 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
490 bool isUnneededShiftMask(
SDNode *
N,
unsigned Width)
const {
492 const APInt &Val =
N->getConstantOperandAPInt(1);
497 APInt Mask = Val | CurDAG->computeKnownBits(
N->getOperand(0)).Zero;
498 return Mask.countr_one() >= Width;
504 SDNode *getGlobalBaseReg();
515 return Subtarget->getInstrInfo();
528 bool isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const;
532 if (!
N->isNonTemporal())
535 unsigned StoreSize =
N->getMemoryVT().getStoreSize();
537 if (
N->getAlign().value() < StoreSize)
546 return Subtarget->hasSSE41();
548 return Subtarget->hasAVX2();
550 return Subtarget->hasAVX512();
554 bool foldLoadStoreIntoMemOperand(
SDNode *
Node);
557 bool shrinkAndImmediate(
SDNode *
N);
558 bool isMaskZeroExtended(
SDNode *
N)
const;
559 bool tryShiftAmountMod(
SDNode *
N);
560 bool tryShrinkShlLogicImm(
SDNode *
N);
566 bool tryMatchBitSelect(
SDNode *
N);
568 MachineSDNode *emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
570 MachineSDNode *emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
bool MayFoldLoad,
574 bool tryOptimizeRem8Extend(
SDNode *
N);
576 bool onlyUsesZeroFlag(
SDValue Flags)
const;
577 bool hasNoSignFlagUses(
SDValue Flags)
const;
578 bool hasNoCarryFlagUses(
SDValue Flags)
const;
582char X86DAGToDAGISel::ID = 0;
589 unsigned Opcode =
N->getOpcode();
596 EVT OpVT =
N->getOperand(0).getValueType();
600 OpVT =
N->getOperand(1).getValueType();
602 return Subtarget->hasVLX();
616bool X86DAGToDAGISel::isMaskZeroExtended(
SDNode *
N)
const {
629 if (OptLevel == CodeGenOptLevel::None)
639 if (useNonTemporalLoad(cast<LoadSDNode>(
N)))
644 switch (
U->getOpcode()) {
670 if (
auto *Imm = dyn_cast<ConstantSDNode>(Op1)) {
671 if (
Imm->getAPIntValue().isSignedIntN(8))
680 Imm->getAPIntValue().getBitWidth() == 64 &&
681 Imm->getAPIntValue().isIntN(32))
688 (
Imm->getAPIntValue() == UINT8_MAX ||
689 Imm->getAPIntValue() == UINT16_MAX ||
690 Imm->getAPIntValue() == UINT32_MAX))
696 (-
Imm->getAPIntValue()).isSignedIntN(8))
700 (-
Imm->getAPIntValue()).isSignedIntN(8) &&
701 hasNoCarryFlagUses(
SDValue(U, 1)))
726 if (
U->getOperand(0).getOpcode() ==
ISD::SHL &&
730 if (
U->getOperand(1).getOpcode() ==
ISD::SHL &&
738 auto *
C = dyn_cast<ConstantSDNode>(U0.
getOperand(0));
739 if (
C &&
C->getSExtValue() == -2)
744 auto *
C = dyn_cast<ConstantSDNode>(U1.
getOperand(0));
745 if (
C &&
C->getSExtValue() == -2)
759 if (isa<ConstantSDNode>(
U->getOperand(1)))
780bool X86DAGToDAGISel::isProfitableToFormMaskedOp(
SDNode *
N)
const {
783 "Unexpected opcode!");
788 return N->getOperand(1).hasOneUse();
797 if (Chain.
getNode() == Load.getNode())
801 "Unexpected chain operand");
815 Load.getOperand(1), Load.getOperand(2));
819 Ops.
append(Call->op_begin() + 1, Call->op_end());
833 if (Callee.getNode() == Chain.
getNode() || !Callee.hasOneUse())
835 auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
853 if (isa<MemSDNode>(Chain.
getNode()) &&
854 cast<MemSDNode>(Chain.
getNode())->writeMem())
860 Callee.getValue(1).hasOneUse())
868 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
871 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
872 0x65, 0x66, 0x67, 0xf0, 0xf2};
875 uint8_t Byte = (Imm >> i) & 0xFF;
887 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
890void X86DAGToDAGISel::PreprocessISelDAG() {
891 bool MadeChange =
false;
893 E = CurDAG->allnodes_end();
I != E; ) {
912 MVT VT =
N->getSimpleValueType(0);
913 int64_t
Imm = cast<ConstantSDNode>(
N)->getSExtValue();
914 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
921 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT,
false,
true);
922 Complement = CurDAG->getNOT(dl, Complement, VT);
924 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Complement);
934 if (
N->getOpcode() ==
X86ISD::AND && !
N->hasAnyUseOfValue(1)) {
936 N->getOperand(0),
N->getOperand(1));
938 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
962 auto mayPreventLoadFold = [&]() {
964 N->getOpcode() ==
ISD::ADD && Subtarget->hasAVX() &&
965 !
N->getOperand(1).hasOneUse();
968 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
974 MVT VT =
N->getSimpleValueType(0);
982 CurDAG->getNode(NewOpcode,
DL, VT,
N->getOperand(0),
AllOnes);
984 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
991 switch (
N->getOpcode()) {
993 MVT VT =
N->getSimpleValueType(0);
995 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1002 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1005 CurDAG->getIntPtrConstant(
Index, dl));
1008 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1017 MVT VT =
N->getSimpleValueType(0);
1019 if (!Subtarget->hasBWI() &&
needBWI(VT)) {
1021 auto *MemNode = cast<MemSDNode>(
N);
1023 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1024 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1025 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1027 MemNode->getMemOperand());
1030 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1033 CurDAG->getIntPtrConstant(
Index, dl));
1037 CurDAG->ReplaceAllUsesWith(
N, To);
1048 auto *Ld = cast<LoadSDNode>(
N);
1049 MVT VT =
N->getSimpleValueType(0);
1057 SDValue Chain = Ld->getChain();
1059 auto *UserLd = dyn_cast<LoadSDNode>(
User);
1060 MVT UserVT =
User->getSimpleValueType(0);
1062 UserLd->getBasePtr() ==
Ptr && UserLd->getChain() == Chain &&
1063 !
User->hasAnyUseOfValue(1) &&
1077 CurDAG->getIntPtrConstant(0, dl));
1078 SDValue Res = CurDAG->getBitcast(VT, Extract);
1082 CurDAG->ReplaceAllUsesWith(
N, To);
1091 EVT EleVT =
N->getOperand(0).getValueType().getVectorElementType();
1092 if (EleVT == MVT::i1)
1095 assert(Subtarget->hasSSE41() &&
"Expected SSE4.1 support!");
1096 assert(
N->getValueType(0).getVectorElementType() != MVT::i16 &&
1097 "We can't replace VSELECT with BLENDV in vXi16!");
1099 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(
N->getOperand(0)) ==
1102 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1103 CurDAG->getTargetConstant(0xCA,
SDLoc(
N), MVT::i8));
1106 N->getOperand(0),
N->getOperand(1),
1110 CurDAG->ReplaceAllUsesWith(
N,
R.getNode());
1123 if (!
N->getSimpleValueType(0).isVector())
1127 switch (
N->getOpcode()) {
1137 if (
N->isStrictFPOpcode())
1139 CurDAG->getNode(NewOpc,
SDLoc(
N), {
N->getValueType(0), MVT::Other},
1140 {
N->getOperand(0),
N->getOperand(1)});
1143 CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1146 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1156 if (!
N->getValueType(0).isVector())
1160 switch (
N->getOpcode()) {
1166 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1167 N->getOperand(0),
N->getOperand(1));
1169 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1178 if (!
N->getValueType(0).isVector())
1182 if (
N->getOperand(0).getScalarValueSizeInBits() == 1) {
1184 "Unexpected opcode for mask vector!");
1192 SDValue Res = CurDAG->getNode(NewOpc,
SDLoc(
N),
N->getValueType(0),
1195 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1215 switch (
N->getOpcode()) {
1231 bool IsStrict =
N->isStrictFPOpcode();
1235 {
N->getValueType(0), MVT::Other},
1236 {
N->getOperand(0),
N->getOperand(1),
1237 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1241 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1243 CurDAG->ReplaceAllUsesWith(
N, Res.
getNode());
1254 MVT VT =
N->getSimpleValueType(0);
1255 if (VT.
isVector() || VT == MVT::f128)
1258 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1259 : VT == MVT::f32 ? MVT::v4f32
1269 if (Subtarget->hasSSE2()) {
1274 switch (
N->getOpcode()) {
1281 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1284 Res = CurDAG->getNode(
N->getOpcode(), dl, VecVT, Op0, Op1);
1287 CurDAG->getIntPtrConstant(0, dl));
1289 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Res);
1296 if (OptLevel != CodeGenOptLevel::None &&
1299 !Subtarget->useIndirectThunkCalls() &&
1300 ((
N->getOpcode() ==
X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1302 (Subtarget->is64Bit() ||
1303 !getTargetMachine().isPositionIndependent())))) {
1342 switch (
N->getOpcode()) {
1347 MVT SrcVT =
N->getOperand(0).getSimpleValueType();
1348 MVT DstVT =
N->getSimpleValueType(0);
1360 if (SrcIsSSE && DstIsSSE)
1363 if (!SrcIsSSE && !DstIsSSE) {
1368 if (
N->getConstantOperandVal(1))
1376 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1377 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1385 CurDAG->getEntryNode(), dl,
N->getOperand(0), MemTmp, MPI, MemVT);
1387 MemTmp, MPI, MemVT);
1394 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(
N, 0), Result);
1403 MVT SrcVT =
N->getOperand(1).getSimpleValueType();
1404 MVT DstVT =
N->getSimpleValueType(0);
1416 if (SrcIsSSE && DstIsSSE)
1419 if (!SrcIsSSE && !DstIsSSE) {
1424 if (
N->getConstantOperandVal(2))
1432 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1433 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
1443 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1444 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), MemTmp};
1448 if (
N->getFlags().hasNoFPExcept()) {
1450 Flags.setNoFPExcept(
true);
1451 Store->setFlags(Flags);
1454 assert(SrcVT == MemVT &&
"Unexpected VT!");
1455 Store = CurDAG->getStore(
N->getOperand(0), dl,
N->getOperand(1), MemTmp,
1460 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1462 Result = CurDAG->getMemIntrinsicNode(
1465 if (
N->getFlags().hasNoFPExcept()) {
1467 Flags.setNoFPExcept(
true);
1471 assert(DstVT == MemVT &&
"Unexpected VT!");
1472 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1480 CurDAG->ReplaceAllUsesWith(
N,
Result.getNode());
1494 CurDAG->RemoveDeadNodes();
1498bool X86DAGToDAGISel::tryOptimizeRem8Extend(
SDNode *
N) {
1499 unsigned Opc =
N->getMachineOpcode();
1500 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1501 Opc != X86::MOVSX64rr8)
1513 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1514 : X86::MOVSX32rr8_NOREX;
1519 if (Opc == X86::MOVSX64rr8) {
1524 ReplaceUses(
N, Extend);
1533void X86DAGToDAGISel::PostprocessISelDAG() {
1535 if (
TM.getOptLevel() == CodeGenOptLevel::None)
1540 bool MadeChange =
false;
1541 while (Position != CurDAG->allnodes_begin()) {
1544 if (
N->use_empty() || !
N->isMachineOpcode())
1547 if (tryOptimizeRem8Extend(
N)) {
1552 unsigned Opc =
N->getMachineOpcode();
1560 case X86::TEST64rr: {
1561 auto &Op0 =
N->getOperand(0);
1566#define CASE_ND(OP) \
1569 switch (
And.getMachineOpcode()) {
1576 if (
And->hasAnyUseOfValue(1))
1579 Opc,
SDLoc(
N), MVT::i32,
And.getOperand(0),
And.getOperand(1));
1580 ReplaceUses(
N,
Test);
1588 if (
And->hasAnyUseOfValue(1))
1591#define FROM_TO(A, B) \
1592 CASE_ND(A) NewOpc = X86::B; \
1594 switch (
And.getMachineOpcode()) {
1604 And.getOperand(3),
And.getOperand(4),
1605 And.getOperand(5),
And.getOperand(0),
1606 And.getOperand(6) };
1608 NewOpc,
SDLoc(
N), MVT::i32, MVT::Other, Ops);
1609 CurDAG->setNodeMemRefs(
1610 Test, cast<MachineSDNode>(
And.getNode())->memoperands());
1622 case X86::KORTESTBrr:
1623 case X86::KORTESTWrr:
1624 case X86::KORTESTDrr:
1625 case X86::KORTESTQrr: {
1627 if (Op0 !=
N->getOperand(1) || !
N->isOnlyUserOf(Op0.
getNode()) ||
1642#define FROM_TO(A, B) \
1654 if (NewOpc == X86::KTESTWrr && !Subtarget->hasDQI())
1659 ReplaceUses(
N, KTest);
1664 case TargetOpcode::SUBREG_TO_REG: {
1665 unsigned SubRegIdx =
N->getConstantOperandVal(2);
1666 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1683 CASE(VMOVAPDZ128rr)
CASE(VMOVUPDZ128rr)
1684 CASE(VMOVAPSZ128rr)
CASE(VMOVUPSZ128rr)
1685 CASE(VMOVDQA32Z128rr)
CASE(VMOVDQU32Z128rr)
1686 CASE(VMOVDQA64Z128rr)
CASE(VMOVDQU64Z128rr)
1687 CASE(VMOVAPDZ256rr)
CASE(VMOVUPDZ256rr)
1688 CASE(VMOVAPSZ256rr)
CASE(VMOVUPSZ256rr)
1689 CASE(VMOVDQA32Z256rr)
CASE(VMOVDQU32Z256rr)
1690 CASE(VMOVDQA64Z256rr)
CASE(VMOVDQU64Z256rr)
1695 if (!
In.isMachineOpcode() ||
1696 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1701 uint64_t TSFlags = getInstrInfo()->get(
In.getMachineOpcode()).TSFlags;
1709 CurDAG->UpdateNodeOperands(
N,
N->getOperand(0), In,
N->getOperand(2));
1716 CurDAG->RemoveDeadNodes();
1721void X86DAGToDAGISel::emitSpecialCodeForMain() {
1722 if (Subtarget->isTargetCygMing()) {
1724 auto &
DL = CurDAG->getDataLayout();
1727 CLI.setChain(CurDAG->getRoot())
1729 CurDAG->getExternalSymbol(
"__main", TLI->getPointerTy(
DL)),
1733 CurDAG->setRoot(
Result.second);
1737void X86DAGToDAGISel::emitFunctionEntryCode() {
1740 if (
F.hasExternalLinkage() &&
F.getName() ==
"main")
1741 emitSpecialCodeForMain();
1751 return isInt<31>(Val);
1755 X86ISelAddressMode &AM) {
1760 int64_t Val = AM.Disp +
Offset;
1763 if (Val != 0 && (AM.ES || AM.MCSym))
1767 if (Subtarget->is64Bit()) {
1770 AM.hasSymbolicDisplacement()))
1774 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1793 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
1794 !AM.hasBaseOrIndexReg())
1801bool X86DAGToDAGISel::matchLoadInAddress(
LoadSDNode *
N, X86ISelAddressMode &AM,
1802 bool AllowSegmentRegForX32) {
1814 if (
isNullConstant(Address) && AM.Segment.getNode() ==
nullptr &&
1815 !IndirectTlsSegRefs &&
1816 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1817 Subtarget->isTargetFuchsia())) {
1818 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1820 switch (
N->getPointerInfo().getAddrSpace()) {
1822 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1825 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1838bool X86DAGToDAGISel::matchWrapper(
SDValue N, X86ISelAddressMode &AM) {
1841 if (AM.hasSymbolicDisplacement())
1844 bool IsRIPRelTLS =
false;
1862 if (IsRIPRel && AM.hasBaseOrIndexReg())
1866 X86ISelAddressMode Backup = AM;
1870 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(N0)) {
1871 AM.GV =
G->getGlobal();
1872 AM.SymbolFlags =
G->getTargetFlags();
1874 }
else if (
auto *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
1875 AM.CP =
CP->getConstVal();
1876 AM.Alignment =
CP->getAlign();
1877 AM.SymbolFlags =
CP->getTargetFlags();
1879 }
else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
1880 AM.ES = S->getSymbol();
1881 AM.SymbolFlags = S->getTargetFlags();
1882 }
else if (
auto *S = dyn_cast<MCSymbolSDNode>(N0)) {
1883 AM.MCSym = S->getMCSymbol();
1884 }
else if (
auto *J = dyn_cast<JumpTableSDNode>(N0)) {
1885 AM.JT = J->getIndex();
1886 AM.SymbolFlags = J->getTargetFlags();
1887 }
else if (
auto *BA = dyn_cast<BlockAddressSDNode>(N0)) {
1888 AM.BlockAddr = BA->getBlockAddress();
1889 AM.SymbolFlags = BA->getTargetFlags();
1890 Offset = BA->getOffset();
1895 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1896 TM.isLargeGlobalValue(AM.GV)) {
1901 if (foldOffsetIntoAddress(
Offset, AM)) {
1907 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1915bool X86DAGToDAGISel::matchAddress(
SDValue N, X86ISelAddressMode &AM) {
1916 if (matchAddressRecursively(
N, AM, 0))
1923 if (Subtarget->isTarget64BitILP32() &&
1924 AM.BaseType == X86ISelAddressMode::RegBase &&
1925 AM.Base_Reg.getNode() !=
nullptr && AM.IndexReg.getNode() ==
nullptr) {
1926 SDValue Save_Base_Reg = AM.Base_Reg;
1927 if (
auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) {
1929 if (matchLoadInAddress(LoadN, AM,
true))
1930 AM.Base_Reg = Save_Base_Reg;
1936 if (AM.Scale == 2 &&
1937 AM.BaseType == X86ISelAddressMode::RegBase &&
1938 AM.Base_Reg.getNode() ==
nullptr) {
1939 AM.Base_Reg = AM.IndexReg;
1946 (!AM.GV || !
TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
1947 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
1948 AM.Base_Reg.getNode() ==
nullptr && AM.IndexReg.getNode() ==
nullptr &&
1950 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1956bool X86DAGToDAGISel::matchAdd(
SDValue &
N, X86ISelAddressMode &AM,
1962 X86ISelAddressMode Backup = AM;
1963 if (!matchAddressRecursively(
N.getOperand(0), AM,
Depth+1) &&
1964 !matchAddressRecursively(Handle.getValue().getOperand(1), AM,
Depth+1))
1969 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
1971 !matchAddressRecursively(Handle.getValue().getOperand(0), AM,
Depth + 1))
1978 if (AM.BaseType == X86ISelAddressMode::RegBase &&
1979 !AM.Base_Reg.getNode() &&
1980 !AM.IndexReg.getNode()) {
1981 N = Handle.getValue();
1982 AM.Base_Reg =
N.getOperand(0);
1983 AM.IndexReg =
N.getOperand(1);
1987 N = Handle.getValue();
1997 if (
N->getNodeId() == -1 ||
2017 X86ISelAddressMode &AM) {
2024 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2025 Mask != (0xffu << ScaleLog))
2028 MVT XVT =
X.getSimpleValueType();
2029 MVT VT =
N.getSimpleValueType();
2054 AM.Scale = (1 << ScaleLog);
2062 X86ISelAddressMode &AM) {
2068 int64_t Mask = cast<ConstantSDNode>(
N->getOperand(1))->getSExtValue();
2073 bool FoundAnyExtend =
false;
2077 FoundAnyExtend =
true;
2095 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2098 MVT VT =
N.getSimpleValueType();
2100 if (FoundAnyExtend) {
2121 AM.Scale = 1 << ShiftAmt;
2122 AM.IndexReg = NewAnd;
2156 X86ISelAddressMode &AM) {
2162 unsigned MaskIdx, MaskLen;
2165 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2171 unsigned AMShiftAmt = MaskIdx;
2175 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2179 unsigned ScaleDown = (64 -
X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2180 if (MaskLZ < ScaleDown)
2182 MaskLZ -= ScaleDown;
2190 bool ReplacingAnyExtend =
false;
2192 unsigned ExtendBits =
X.getSimpleValueType().getSizeInBits() -
2193 X.getOperand(0).getSimpleValueType().getSizeInBits();
2196 X =
X.getOperand(0);
2197 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2198 ReplacingAnyExtend =
true;
2200 APInt MaskedHighBits =
2207 MVT VT =
N.getSimpleValueType();
2208 if (ReplacingAnyExtend) {
2209 assert(
X.getValueType() != VT);
2216 MVT XVT =
X.getSimpleValueType();
2237 AM.Scale = 1 << AMShiftAmt;
2238 AM.IndexReg = NewExt;
2248 X86ISelAddressMode &AM,
2256 if (!Subtarget.hasTBM() &&
2257 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2261 unsigned MaskIdx, MaskLen;
2269 unsigned AMShiftAmt = MaskIdx;
2273 if (AMShiftAmt == 0 || AMShiftAmt > 3)
return true;
2275 MVT XVT =
X.getSimpleValueType();
2276 MVT VT =
N.getSimpleValueType();
2301 AM.Scale = 1 << AMShiftAmt;
2302 AM.IndexReg = NewExt;
2309 X86ISelAddressMode &AM,
2311 assert(AM.IndexReg.getNode() ==
nullptr &&
"IndexReg already matched");
2312 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2313 "Illegal index scale");
2319 EVT VT =
N.getValueType();
2320 unsigned Opc =
N.getOpcode();
2323 if (CurDAG->isBaseWithConstantOffset(
N)) {
2324 auto *AddVal = cast<ConstantSDNode>(
N.getOperand(1));
2326 if (!foldOffsetIntoAddress(
Offset, AM))
2327 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2331 if (Opc ==
ISD::ADD &&
N.getOperand(0) ==
N.getOperand(1)) {
2332 if (AM.Scale <= 4) {
2334 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2340 uint64_t ShiftAmt =
N.getConstantOperandVal(1);
2341 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2342 if ((AM.Scale * ScaleAmt) <= 8) {
2343 AM.Scale *= ScaleAmt;
2344 return matchIndexRecursively(
N.getOperand(0), AM,
Depth + 1);
2352 if (Src.getOpcode() ==
ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2354 if (CurDAG->isBaseWithConstantOffset(Src)) {
2355 SDValue AddSrc = Src.getOperand(0);
2356 auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1));
2358 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2360 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2366 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2367 CurDAG->RemoveDeadNode(
N.getNode());
2379 unsigned SrcOpc = Src.getOpcode();
2380 if (((SrcOpc ==
ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2381 CurDAG->isADDLike(Src,
true)) &&
2383 if (CurDAG->isBaseWithConstantOffset(Src)) {
2384 SDValue AddSrc = Src.getOperand(0);
2386 if (!foldOffsetIntoAddress(
Offset * AM.Scale, AM)) {
2397 if ((AM.Scale * ScaleAmt) <= 8 &&
2399 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2400 AM.Scale *= ScaleAmt;
2401 SDValue ExtShVal = CurDAG->getNode(Opc,
DL, VT, ShVal);
2410 SDValue ExtSrc = CurDAG->getNode(Opc,
DL, VT, AddSrc);
2412 SDValue ExtAdd = CurDAG->getNode(SrcOpc,
DL, VT, ExtSrc, ExtVal);
2416 CurDAG->ReplaceAllUsesWith(
N, ExtAdd);
2417 CurDAG->RemoveDeadNode(
N.getNode());
2418 return Res ? Res : ExtSrc;
2428bool X86DAGToDAGISel::matchAddressRecursively(
SDValue N, X86ISelAddressMode &AM,
2432 dbgs() <<
"MatchAddress: ";
2437 return matchAddressBase(
N, AM);
2442 if (AM.isRIPRelative()) {
2446 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2449 if (
auto *Cst = dyn_cast<ConstantSDNode>(
N))
2450 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2455 switch (
N.getOpcode()) {
2458 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2459 if (
const auto *ESNode = dyn_cast<MCSymbolSDNode>(
N.getOperand(0))) {
2461 AM.MCSym = ESNode->getMCSymbol();
2467 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2468 if (!foldOffsetIntoAddress(Val, AM))
2475 if (!matchWrapper(
N, AM))
2480 if (!matchLoadInAddress(cast<LoadSDNode>(
N), AM))
2485 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2486 AM.Base_Reg.getNode() ==
nullptr &&
2488 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2489 AM.Base_FrameIndex = cast<FrameIndexSDNode>(
N)->getIndex();
2495 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2498 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
2499 unsigned Val = CN->getZExtValue();
2504 if (Val == 1 || Val == 2 || Val == 3) {
2506 AM.Scale = 1 << Val;
2507 AM.IndexReg = matchIndexRecursively(ShVal, AM,
Depth + 1);
2515 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2519 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2520 "Unexpected value size!");
2529 if (!isa<ConstantSDNode>(
N.getOperand(1)) ||
2530 !isa<ConstantSDNode>(
And.getOperand(1)))
2532 uint64_t Mask =
And.getConstantOperandVal(1) >>
N.getConstantOperandVal(1);
2544 if (
N.getResNo() != 0)
break;
2549 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2550 AM.Base_Reg.getNode() ==
nullptr &&
2551 AM.IndexReg.getNode() ==
nullptr) {
2552 if (
auto *CN = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
2553 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2554 CN->getZExtValue() == 9) {
2555 AM.Scale =
unsigned(CN->getZExtValue())-1;
2566 auto *AddVal = cast<ConstantSDNode>(MulVal.
getOperand(1));
2567 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2568 if (foldOffsetIntoAddress(Disp, AM))
2569 Reg =
N.getOperand(0);
2571 Reg =
N.getOperand(0);
2574 AM.IndexReg = AM.Base_Reg =
Reg;
2593 X86ISelAddressMode Backup = AM;
2594 if (matchAddressRecursively(
N.getOperand(0), AM,
Depth+1)) {
2595 N = Handle.getValue();
2599 N = Handle.getValue();
2601 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2616 RHS.getOperand(0).getValueType() == MVT::i32))
2620 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2621 !AM.Base_Reg.getNode()->hasOneUse()) ||
2622 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2626 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2627 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2628 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2640 AM.NegateIndex =
true;
2648 if (!CurDAG->isADDLike(
N))
2652 if (!matchAdd(
N, AM,
Depth))
2661 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
break;
2665 assert(
N.getSimpleValueType().getSizeInBits() <= 64 &&
2666 "Unexpected value size!");
2668 if (!isa<ConstantSDNode>(
N.getOperand(1)))
2671 if (
N.getOperand(0).getOpcode() ==
ISD::SRL) {
2700 if (AM.IndexReg.getNode() !=
nullptr || AM.Scale != 1)
2710 AM.IndexReg =
Index;
2716 if (Src.getOpcode() ==
ISD::AND && Src.hasOneUse())
2717 if (
auto *MaskC = dyn_cast<ConstantSDNode>(Src.getOperand(1))) {
2718 Mask = MaskC->getAPIntValue();
2719 Src = Src.getOperand(0);
2722 if (Src.getOpcode() ==
ISD::SHL && Src.hasOneUse()) {
2724 SDValue ShlSrc = Src.getOperand(0);
2725 SDValue ShlAmt = Src.getOperand(1);
2726 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShlAmt);
2729 unsigned ShAmtV = ShAmtC->getZExtValue();
2737 if (!Src->getFlags().hasNoUnsignedWrap() &&
2738 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2746 MVT VT =
N.getSimpleValueType();
2750 if (!
Mask.isAllOnes()) {
2751 Res = CurDAG->getConstant(
Mask.lshr(ShAmtV),
DL, SrcVT);
2753 Res = CurDAG->getNode(
ISD::AND,
DL, SrcVT, ShlSrc, Res);
2760 CurDAG->ReplaceAllUsesWith(
N, NewShl);
2761 CurDAG->RemoveDeadNode(
N.getNode());
2764 AM.Scale = 1 << ShAmtV;
2768 AM.IndexReg = matchIndexRecursively(Zext, AM,
Depth + 1);
2772 if (Src.getOpcode() ==
ISD::SRL && !
Mask.isAllOnes()) {
2775 Src.getOperand(0), AM))
2780 Src.getOperand(0), AM))
2785 Src.getOperand(0), AM, *Subtarget))
2793 return matchAddressBase(
N, AM);
2798bool X86DAGToDAGISel::matchAddressBase(
SDValue N, X86ISelAddressMode &AM) {
2800 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2802 if (!AM.IndexReg.getNode()) {
2813 AM.BaseType = X86ISelAddressMode::RegBase;
2818bool X86DAGToDAGISel::matchVectorAddressRecursively(
SDValue N,
2819 X86ISelAddressMode &AM,
2823 dbgs() <<
"MatchVectorAddress: ";
2828 return matchAddressBase(
N, AM);
2831 switch (
N.getOpcode()) {
2833 uint64_t Val = cast<ConstantSDNode>(
N)->getSExtValue();
2834 if (!foldOffsetIntoAddress(Val, AM))
2839 if (!matchWrapper(
N, AM))
2847 X86ISelAddressMode Backup = AM;
2848 if (!matchVectorAddressRecursively(
N.getOperand(0), AM,
Depth + 1) &&
2849 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2855 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2857 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2862 N = Handle.getValue();
2867 return matchAddressBase(
N, AM);
2873bool X86DAGToDAGISel::matchVectorAddress(
SDValue N, X86ISelAddressMode &AM) {
2874 return matchVectorAddressRecursively(
N, AM, 0);
2882 X86ISelAddressMode AM;
2888 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2890 AM.IndexReg = IndexOp;
2894 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2896 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2898 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2904 if (matchVectorAddress(BasePtr, AM))
2907 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2921 X86ISelAddressMode AM;
2933 unsigned AddrSpace =
2934 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
2936 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2938 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2940 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2945 MVT VT =
N.getSimpleValueType();
2947 if (matchAddress(
N, AM))
2950 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
2966 N =
N.getOperand(0);
2981 const GlobalValue *GV = cast<GlobalAddressSDNode>(
N)->getGlobal();
2983 return CR->getUnsignedMax().ult(1ull << 32);
2985 return !
TM.isLargeGlobalValue(GV);
2994 if (!selectLEAAddr(
N,
Base, Scale,
Index, Disp, Segment))
2997 auto *
RN = dyn_cast<RegisterSDNode>(
Base);
2998 if (RN &&
RN->getReg() == 0)
2999 Base = CurDAG->getRegister(0, MVT::i64);
3000 else if (
Base.getValueType() == MVT::i32 && !isa<FrameIndexSDNode>(
Base)) {
3004 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3008 RN = dyn_cast<RegisterSDNode>(
Index);
3009 if (RN &&
RN->getReg() == 0)
3010 Index = CurDAG->getRegister(0, MVT::i64);
3013 "Expect to be extending 32-bit registers for use in LEA");
3016 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit,
DL, MVT::i64, ImplDef,
3025bool X86DAGToDAGISel::selectLEAAddr(
SDValue N,
3029 X86ISelAddressMode AM;
3033 MVT VT =
N.getSimpleValueType();
3038 SDValue T = CurDAG->getRegister(0, MVT::i32);
3040 if (matchAddress(
N, AM))
3045 unsigned Complexity = 0;
3046 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3048 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3051 if (AM.IndexReg.getNode())
3064 if (AM.hasSymbolicDisplacement()) {
3066 if (Subtarget->is64Bit())
3076 auto isMathWithFlags = [](
SDValue V) {
3077 switch (
V.getOpcode()) {
3098 if (isMathWithFlags(
N.getOperand(0)) || isMathWithFlags(
N.getOperand(1)))
3106 if (Complexity <= 2)
3109 getAddressOperands(AM,
DL, VT,
Base, Scale,
Index, Disp, Segment);
3120 X86ISelAddressMode AM;
3121 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N)) {
3122 AM.GV = GA->getGlobal();
3123 AM.Disp += GA->getOffset();
3124 AM.SymbolFlags = GA->getTargetFlags();
3126 auto *SA = cast<ExternalSymbolSDNode>(
N);
3127 AM.ES = SA->getSymbol();
3128 AM.SymbolFlags = SA->getTargetFlags();
3131 if (Subtarget->is32Bit()) {
3133 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3136 MVT VT =
N.getSimpleValueType();
3137 getAddressOperands(AM,
SDLoc(
N), VT,
Base, Scale,
Index, Disp, Segment);
3145 EVT VT =
N.getValueType();
3146 bool WasTruncated =
false;
3148 WasTruncated =
true;
3149 N =
N.getOperand(0);
3158 unsigned Opc =
N.getOperand(0)->getOpcode();
3160 Op =
N.getOperand(0);
3163 return !WasTruncated;
3167 auto *GA = cast<GlobalAddressSDNode>(
N.getOperand(0));
3168 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3169 if (!CR || CR->getUnsignedMax().uge(1ull << VT.
getSizeInBits()))
3173 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(),
SDLoc(
N), VT,
3174 GA->getOffset(), GA->getTargetFlags());
3182 assert(Root &&
P &&
"Unknown root/parent nodes");
3184 !IsProfitableToFold(
N,
P, Root) ||
3185 !IsLegalToFold(
N,
P, Root, OptLevel))
3188 return selectAddr(
N.getNode(),
3189 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3196 assert(Root &&
P &&
"Unknown root/parent nodes");
3198 !IsProfitableToFold(
N,
P, Root) ||
3199 !IsLegalToFold(
N,
P, Root, OptLevel))
3202 return selectAddr(
N.getNode(),
3203 N.getOperand(1),
Base, Scale,
Index, Disp, Segment);
3209SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3210 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3212 return CurDAG->getRegister(GlobalBaseReg, TLI->
getPointerTy(
DL)).getNode();
3215bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(
unsigned Width,
SDNode *
N)
const {
3217 N =
N->getOperand(0).getNode();
3221 auto *GA = dyn_cast<GlobalAddressSDNode>(
N->getOperand(0));
3225 auto *GV = GA->getGlobal();
3228 return CR->getSignedMin().sge(-1ull << Width) &&
3229 CR->getSignedMax().slt(1ull << Width);
3234 return Width == 32 && !
TM.isLargeGlobalValue(GV);
3238 assert(
N->isMachineOpcode() &&
"Unexpected node");
3239 unsigned Opc =
N->getMachineOpcode();
3240 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3245 return static_cast<X86::CondCode>(
N->getConstantOperandVal(CondNo));
3250bool X86DAGToDAGISel::onlyUsesZeroFlag(
SDValue Flags)
const {
3255 if (UI.getUse().getResNo() !=
Flags.getResNo())
3259 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3263 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3265 if (FlagUI.getUse().getResNo() != 1)
continue;
3267 if (!FlagUI->isMachineOpcode())
return false;
3286bool X86DAGToDAGISel::hasNoSignFlagUses(
SDValue Flags)
const {
3291 if (UI.getUse().getResNo() !=
Flags.getResNo())
3295 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3299 FlagUE = UI->
use_end(); FlagUI != FlagUE; ++FlagUI) {
3301 if (FlagUI.getUse().getResNo() != 1)
continue;
3303 if (!FlagUI->isMachineOpcode())
return false;
3342 bool X86DAGToDAGISel::hasNoCarryFlagUses(
SDValue Flags)
const {
3347 if (UI.getUse().getResNo() !=
Flags.getResNo())
3350 unsigned UIOpc = UI->getOpcode();
3354 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
3358 FlagUI != FlagUE; ++FlagUI) {
3360 if (FlagUI.getUse().getResNo() != 1)
3363 if (!FlagUI->isMachineOpcode())
3404 if (StoredVal.
getResNo() != 0)
return false;
3418 LoadNode = cast<LoadSDNode>(Load);
3421 if (!Load.hasOneUse())
3429 bool FoundLoad =
false;
3433 const unsigned int Max = 1024;
3475 if (Chain == Load.getValue(1)) {
3481 if (
Op == Load.getValue(1)) {
3497 if (
Op.getNode() != LoadNode)
3530bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(
SDNode *
Node) {
3531 auto *StoreNode = cast<StoreSDNode>(
Node);
3538 EVT MemVT = StoreNode->getMemoryVT();
3539 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3543 bool IsCommutable =
false;
3544 bool IsNegate =
false;
3558 IsCommutable =
true;
3562 unsigned LoadOpNo = IsNegate ? 1 : 0;
3566 LoadNode, InputChain)) {
3573 LoadNode, InputChain))
3582 auto SelectOpcode = [&](
unsigned Opc64,
unsigned Opc32,
unsigned Opc16,
3603 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3613 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3617 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3620 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3621 : SelectOpcode(
X86::DEC64m,
X86::DEC32m,
X86::DEC16m,
X86::DEC8m);
3634 auto SelectRegOpcode = [SelectOpcode](
unsigned Opc) {
3637 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3640 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3643 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3646 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3649 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3652 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3654 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3660 auto SelectImmOpcode = [SelectOpcode](
unsigned Opc) {
3663 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3666 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3669 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3672 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3675 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3678 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3681 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3688 unsigned NewOpc = SelectRegOpcode(Opc);
3693 if (
auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
3694 int64_t OperandV = OperandC->getSExtValue();
3700 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) ||
3701 (MemVT == MVT::i64 && !isInt<32>(OperandV) &&
3702 isInt<32>(-OperandV))) &&
3703 hasNoCarryFlagUses(StoredVal.
getValue(1))) {
3704 OperandV = -OperandV;
3708 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3709 Operand = CurDAG->getTargetConstant(OperandV,
SDLoc(
Node), MemVT);
3710 NewOpc = SelectImmOpcode(Opc);
3716 CurDAG->getCopyToReg(InputChain,
SDLoc(
Node), X86::EFLAGS,
3720 Segment, Operand, CopyTo, CopyTo.
getValue(1)};
3721 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3725 Segment, Operand, InputChain};
3726 Result = CurDAG->getMachineNode(NewOpc,
SDLoc(
Node), MVT::i32, MVT::Other,
3737 CurDAG->setNodeMemRefs(Result, MemOps);
3743 CurDAG->RemoveDeadNode(
Node);
3754bool X86DAGToDAGISel::matchBitExtract(
SDNode *
Node) {
3758 "Should be either an and-mask, or right-shift after clearing high bits.");
3761 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3764 MVT NVT =
Node->getSimpleValueType(0);
3767 if (NVT != MVT::i32 && NVT != MVT::i64)
3775 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3776 auto checkUses = [AllowExtraUsesByDefault](
3778 std::optional<bool> AllowExtraUses) {
3779 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3780 Op.getNode()->hasNUsesOfValue(NUses,
Op.getResNo());
3782 auto checkOneUse = [checkUses](
SDValue Op,
3783 std::optional<bool> AllowExtraUses =
3785 return checkUses(
Op, 1, AllowExtraUses);
3787 auto checkTwoUse = [checkUses](
SDValue Op,
3788 std::optional<bool> AllowExtraUses =
3790 return checkUses(
Op, 2, AllowExtraUses);
3793 auto peekThroughOneUseTruncation = [checkOneUse](
SDValue V) {
3795 assert(
V.getSimpleValueType() == MVT::i32 &&
3796 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3797 "Expected i64 -> i32 truncation");
3798 V =
V.getOperand(0);
3804 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3807 if (
Mask->getOpcode() !=
ISD::ADD || !checkOneUse(Mask))
3813 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3818 NBits =
M0->getOperand(1);
3819 NegateNBits =
false;
3823 auto isAllOnes = [
this, peekThroughOneUseTruncation, NVT](
SDValue V) {
3824 V = peekThroughOneUseTruncation(V);
3825 return CurDAG->MaskedValueIsAllOnes(
3831 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3834 if (
Mask.getOpcode() !=
ISD::XOR || !checkOneUse(Mask))
3837 if (!isAllOnes(
Mask->getOperand(1)))
3840 SDValue M0 = peekThroughOneUseTruncation(
Mask->getOperand(0));
3844 if (!isAllOnes(
M0->getOperand(0)))
3846 NBits =
M0->getOperand(1);
3847 NegateNBits =
false;
3853 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](
SDValue ShiftAmt,
3854 unsigned Bitwidth) {
3859 NBits = NBits.getOperand(0);
3864 auto *V0 = dyn_cast<ConstantSDNode>(NBits.getOperand(0));
3865 if (!V0 || V0->getZExtValue() != Bitwidth)
3867 NBits = NBits.getOperand(1);
3868 NegateNBits =
false;
3874 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3877 Mask = peekThroughOneUseTruncation(Mask);
3878 unsigned Bitwidth =
Mask.getSimpleValueType().getSizeInBits();
3880 if (
Mask.getOpcode() !=
ISD::SRL || !checkOneUse(Mask))
3887 if (!checkOneUse(
M1))
3889 canonicalizeShiftAmt(
M1, Bitwidth);
3894 return !NegateNBits;
3902 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3903 AllowExtraUsesByDefault, &NegateNBits,
3916 canonicalizeShiftAmt(N1, Bitwidth);
3920 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
3921 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
3927 auto matchLowBitMask = [matchPatternA, matchPatternB,
3929 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
3933 X =
Node->getOperand(0);
3936 if (matchLowBitMask(Mask)) {
3940 if (!matchLowBitMask(Mask))
3944 X = CurDAG->getAllOnesConstant(
SDLoc(
Node), NVT);
3945 }
else if (!matchPatternD(
Node))
3950 if (NegateNBits && !Subtarget->hasBMI2())
3962 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i32), 0);
3965 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit,
DL, MVT::i32);
3967 NBits =
SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG,
DL,
3968 MVT::i32, ImplDef, NBits, SRIdxVal),
3978 NBits = CurDAG->getNode(
ISD::SUB,
DL, MVT::i32, BitWidthC, NBits);
3982 if (Subtarget->hasBMI2()) {
3984 if (NVT != MVT::i32) {
3992 SelectCode(Extract.
getNode());
4001 SDValue RealX = peekThroughOneUseTruncation(
X);
4007 MVT XVT =
X.getSimpleValueType();
4017 SDValue C8 = CurDAG->getConstant(8,
DL, MVT::i8);
4025 SDValue ShiftAmt =
X.getOperand(1);
4026 X =
X.getOperand(0);
4029 "Expected shift amount to be i8");
4033 SDValue OrigShiftAmt = ShiftAmt;
4038 Control = CurDAG->getNode(
ISD::OR,
DL, MVT::i32, Control, ShiftAmt);
4043 if (XVT != MVT::i32) {
4058 SelectCode(Extract.
getNode());
4065 MVT NVT =
Node->getSimpleValueType(0);
4078 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4079 if (!PreferBEXTR && !Subtarget->hasBMI2())
4091 if (NVT != MVT::i32 && NVT != MVT::i64)
4095 auto *MaskCst = dyn_cast<ConstantSDNode>(N1);
4096 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
4097 if (!MaskCst || !ShiftCst)
4105 uint64_t Shift = ShiftCst->getZExtValue();
4110 if (Shift == 8 && MaskSize == 8)
4121 if (!PreferBEXTR && MaskSize <= 32)
4125 unsigned ROpc, MOpc;
4127#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4129 assert(Subtarget->hasBMI2() &&
"We must have BMI2's BZHI then.");
4133 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4138 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4139 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4145 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4146 if (Subtarget->hasTBM()) {
4147 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4148 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4150 assert(Subtarget->hasBMI() &&
"We must have BMI1's BEXTR then.");
4156 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4157 Control =
SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4163 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4164 if (tryFoldLoad(
Node, N0.
getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4166 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.
getOperand(0)};
4167 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4168 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4172 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
4174 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4179 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4183 CurDAG->getMachineNode(NewOpc, dl, NVT,
SDValue(NewNode, 0), ShAmt);
4190MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(
unsigned ROpc,
unsigned MOpc,
4191 bool MayFoldLoad,
const SDLoc &dl,
4196 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4197 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4200 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4201 if (MayFoldLoad && tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4202 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4204 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4205 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4209 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
4214 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4215 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4222MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(
unsigned ROpc,
unsigned MOpc,
4223 bool MayFoldLoad,
const SDLoc &dl,
4229 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue();
4230 Imm = CurDAG->getTargetConstant(*Val,
SDLoc(
Node),
Imm.getValueType());
4233 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4234 if (MayFoldLoad && tryFoldLoad(
Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4235 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
4237 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4238 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4243 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()});
4248 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4249 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4254bool X86DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
4255 EVT VT =
N->getValueType(0);
4262 unsigned Size = VT == MVT::i64 ? 64 : 32;
4264 SDValue OrigShiftAmt =
N->getOperand(1);
4265 SDValue ShiftAmt = OrigShiftAmt;
4280 auto *Add0C = dyn_cast<ConstantSDNode>(Add0);
4281 auto *Add1C = dyn_cast<ConstantSDNode>(Add1);
4284 if (Add1C && Add1C->getAPIntValue().urem(
Size) == 0) {
4288 ((Add0C && Add0C->getAPIntValue().urem(
Size) ==
Size - 1) ||
4289 (Add1C && Add1C->getAPIntValue().urem(
Size) ==
Size - 1))) {
4293 assert(Add0C ==
nullptr || Add1C ==
nullptr);
4302 NewShiftAmt = CurDAG->getNode(
ISD::XOR,
DL, OpVT,
4303 Add0C ==
nullptr ? Add0 : Add1,
AllOnes);
4309 Add0C->getZExtValue() != 0) {
4312 if (Add0C->getZExtValue() %
Size == 0)
4315 Add0C->getZExtValue() % 32 == 0) {
4323 Add0 = CurDAG->getZExtOrTrunc(Add0,
DL, SubVT);
4327 X = CurDAG->getNode(
ISD::ADD,
DL, SubVT, Add1, Add0);
4349 NewShiftAmt = CurDAG->getNode(
ISD::TRUNCATE,
DL, MVT::i8, NewShiftAmt);
4356 NewShiftAmt = CurDAG->getNode(
ISD::AND,
DL, MVT::i8, NewShiftAmt,
4357 CurDAG->getConstant(
Size - 1,
DL, MVT::i8));
4361 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(
N,
N->getOperand(0),
4363 if (UpdatedNode !=
N) {
4366 ReplaceNode(
N, UpdatedNode);
4373 CurDAG->RemoveDeadNode(OrigShiftAmt.
getNode());
4381bool X86DAGToDAGISel::tryShrinkShlLogicImm(
SDNode *
N) {
4382 MVT NVT =
N->getSimpleValueType(0);
4383 unsigned Opcode =
N->getOpcode();
4391 auto *Cst = dyn_cast<ConstantSDNode>(N1);
4395 int64_t Val = Cst->getSExtValue();
4400 bool FoundAnyExtend =
false;
4404 FoundAnyExtend =
true;
4412 if (NVT != MVT::i32 && NVT != MVT::i64)
4415 auto *ShlCst = dyn_cast<ConstantSDNode>(Shift.
getOperand(1));
4419 uint64_t ShAmt = ShlCst->getZExtValue();
4423 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4424 if (Opcode !=
ISD::AND && (Val & RemovedBitsMask) != 0)
4429 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4433 ShiftedVal = (
uint64_t)Val >> ShAmt;
4434 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4437 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4440 ShiftedVal = Val >> ShAmt;
4441 if ((!isInt<8>(Val) && isInt<8>(ShiftedVal)) ||
4442 (!isInt<32>(Val) && isInt<32>(ShiftedVal)))
4446 ShiftedVal = (
uint64_t)Val >> ShAmt;
4447 if (NVT == MVT::i64 && !isUInt<32>(Val) && isUInt<32>(ShiftedVal))
4454 if (!CanShrinkImmediate(ShiftedVal))
4464 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4470 NeededMask &= ~Cst->getAPIntValue();
4472 if (CurDAG->MaskedValueIsZero(
N->getOperand(0), NeededMask))
4477 if (FoundAnyExtend) {
4483 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
4485 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT,
X, NewCst);
4494bool X86DAGToDAGISel::matchVPTERNLOG(
SDNode *Root,
SDNode *ParentA,
4498 assert(
A.isOperandOf(ParentA) &&
B.isOperandOf(ParentB) &&
4499 C.isOperandOf(ParentC) &&
"Incorrect parent node");
4501 auto tryFoldLoadOrBCast =
4504 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4510 L =
L.getOperand(0);
4517 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4518 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4522 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4525 bool FoldedLoad =
false;
4526 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4527 if (tryFoldLoadOrBCast(Root, ParentC,
C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4529 }
else if (tryFoldLoadOrBCast(Root, ParentA,
A, Tmp0, Tmp1, Tmp2, Tmp3,
4534 uint8_t OldImm =
Imm;
4535 Imm = OldImm & 0xa5;
4536 if (OldImm & 0x02)
Imm |= 0x10;
4537 if (OldImm & 0x10)
Imm |= 0x02;
4538 if (OldImm & 0x08)
Imm |= 0x40;
4539 if (OldImm & 0x40)
Imm |= 0x08;
4540 }
else if (tryFoldLoadOrBCast(Root, ParentB,
B, Tmp0, Tmp1, Tmp2, Tmp3,
4545 uint8_t OldImm =
Imm;
4546 Imm = OldImm & 0x99;
4547 if (OldImm & 0x02)
Imm |= 0x04;
4548 if (OldImm & 0x04)
Imm |= 0x02;
4549 if (OldImm & 0x20)
Imm |= 0x40;
4550 if (OldImm & 0x40)
Imm |= 0x20;
4555 SDValue TImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i8);
4561 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4565 auto *MemIntr = cast<MemIntrinsicSDNode>(
C);
4566 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4567 assert((EltSize == 32 || EltSize == 64) &&
"Unexpected broadcast size!");
4569 bool UseD = EltSize == 32;
4571 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4573 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4575 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4581 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4583 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4585 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4590 SDValue Ops[] = {
A,
B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm,
C.getOperand(0)};
4591 MNode = CurDAG->getMachineNode(Opc,
DL, VTs, Ops);
4594 ReplaceUses(
C.getValue(1),
SDValue(MNode, 1));
4596 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(
C)->getMemOperand()});
4601 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4603 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4605 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4609 MNode = CurDAG->getMachineNode(Opc,
DL, NVT, {
A,
B,
C, TImm});
4613 CurDAG->RemoveDeadNode(Root);
4619bool X86DAGToDAGISel::tryVPTERNLOG(
SDNode *
N) {
4620 MVT NVT =
N->getSimpleValueType(0);
4623 if (!NVT.
isVector() || !Subtarget->hasAVX512() ||
4634 auto getFoldableLogicOp = [](
SDValue Op) {
4637 Op =
Op.getOperand(0);
4639 if (!
Op.hasOneUse())
4642 unsigned Opc =
Op.getOpcode();
4651 if ((FoldableOp = getFoldableLogicOp(N1))) {
4653 }
else if ((FoldableOp = getFoldableLogicOp(N0))) {
4666 uint8_t TernlogMagicA = 0xf0;
4667 uint8_t TernlogMagicB = 0xcc;
4668 uint8_t TernlogMagicC = 0xaa;
4677 Parent =
Op.getNode();
4678 Op =
Op.getOperand(0);
4682 PeekThroughNot(
A, ParentA, TernlogMagicA);
4683 PeekThroughNot(
B, ParentB, TernlogMagicB);
4684 PeekThroughNot(
C, ParentC, TernlogMagicC);
4689 case ISD::AND:
Imm = TernlogMagicB & TernlogMagicC;
break;
4690 case ISD::OR:
Imm = TernlogMagicB | TernlogMagicC;
break;
4691 case ISD::XOR:
Imm = TernlogMagicB ^ TernlogMagicC;
break;
4695 switch (
N->getOpcode()) {
4699 Imm &= ~TernlogMagicA;
4701 Imm = ~(
Imm) & TernlogMagicA;
4708 return matchVPTERNLOG(
N, ParentA, ParentB, ParentC,
A,
B,
C, Imm);
4718bool X86DAGToDAGISel::shrinkAndImmediate(
SDNode *
And) {
4721 MVT VT =
And->getSimpleValueType(0);
4722 if (VT != MVT::i32 && VT != MVT::i64)
4725 auto *And1C = dyn_cast<ConstantSDNode>(
And->getOperand(1));
4734 APInt MaskVal = And1C->getAPIntValue();
4736 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4740 if (VT == MVT::i64 && MaskLZ >= 32) {
4742 MaskVal = MaskVal.
trunc(32);
4747 APInt NegMaskVal = MaskVal | HighZeros;
4756 if (VT == MVT::i64 && MaskVal.
getBitWidth() < 64) {
4757 NegMaskVal = NegMaskVal.
zext(64);
4758 HighZeros = HighZeros.
zext(64);
4763 if (!CurDAG->MaskedValueIsZero(And0, HighZeros))
4783 bool FoldedBCast,
bool Masked) {
4784#define VPTESTM_CASE(VT, SUFFIX) \
4787 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4788 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4791#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4792default: llvm_unreachable("Unexpected VT!"); \
4793VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4794VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4795VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4796VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4797VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4798VPTESTM_CASE(v8i64, QZ##SUFFIX)
4800#define VPTESTM_FULL_CASES(SUFFIX) \
4801VPTESTM_BROADCAST_CASES(SUFFIX) \
4802VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4803VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4804VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4805VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4806VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4807VPTESTM_CASE(v32i16, WZ##SUFFIX)
4825#undef VPTESTM_FULL_CASES
4826#undef VPTESTM_BROADCAST_CASES
4832bool X86DAGToDAGISel::tryVPTESTM(
SDNode *Root,
SDValue Setcc,
4834 assert(Subtarget->hasAVX512() &&
"Expected AVX512!");
4884 if (tryFoldLoad(Root,
P, L,
Base, Scale,
Index, Disp, Segment))
4889 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
4895 L =
L.getOperand(0);
4901 auto *MemIntr = cast<MemIntrinsicSDNode>(L);
4902 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.
getSizeInBits())
4905 return tryFoldBroadcast(Root,
P, L,
Base, Scale,
Index, Disp, Segment);
4909 bool CanFoldLoads = Src0 != Src1;
4911 bool FoldedLoad =
false;
4912 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4914 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src1, Tmp0, Tmp1, Tmp2,
4918 FoldedLoad = tryFoldLoadOrBCast(Root, N0.
getNode(), Src0, Tmp0, Tmp1,
4927 bool IsMasked = InMask.
getNode() !=
nullptr;
4940 SDValue ImplDef =
SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4942 Src0 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src0);
4945 Src1 = CurDAG->getTargetInsertSubreg(
SubReg, dl, CmpVT, ImplDef, Src1);
4950 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4951 InMask =
SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4952 dl, MaskVT, InMask, RC), 0);
4957 unsigned Opc =
getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
4962 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
4965 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4967 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4969 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4971 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
4977 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
4980 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
4982 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
4988 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4989 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4990 dl, ResVT,
SDValue(CNode, 0), RC);
4994 CurDAG->RemoveDeadNode(Root);
5000bool X86DAGToDAGISel::tryMatchBitSelect(
SDNode *
N) {
5003 MVT NVT =
N->getSimpleValueType(0);
5006 if (!NVT.
isVector() || !Subtarget->hasAVX512())
5040 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5049 MVT NVT =
Node->getSimpleValueType(0);
5050 unsigned Opcode =
Node->getOpcode();
5053 if (
Node->isMachineOpcode()) {
5055 Node->setNodeId(-1);
5062 unsigned IntNo =
Node->getConstantOperandVal(1);
5065 case Intrinsic::x86_encodekey128:
5066 case Intrinsic::x86_encodekey256: {
5067 if (!Subtarget->hasKL())
5073 case Intrinsic::x86_encodekey128:
5074 Opcode = X86::ENCODEKEY128;
5076 case Intrinsic::x86_encodekey256:
5077 Opcode = X86::ENCODEKEY256;
5082 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(3),
5084 if (Opcode == X86::ENCODEKEY256)
5085 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(4),
5089 Opcode, dl,
Node->getVTList(),
5090 {Node->getOperand(2), Chain, Chain.getValue(1)});
5091 ReplaceNode(
Node, Res);
5094 case Intrinsic::x86_tileloadd64_internal:
5095 case Intrinsic::x86_tileloaddt164_internal: {
5096 if (!Subtarget->hasAMXTILE())
5098 unsigned Opc = IntNo == Intrinsic::x86_tileloadd64_internal
5100 : X86::PTILELOADDT1V;
5103 SDValue Scale = getI8Imm(1, dl);
5105 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5106 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5110 Node->getOperand(3),
5117 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5118 ReplaceNode(
Node, CNode);
5125 unsigned IntNo =
Node->getConstantOperandVal(1);
5128 case Intrinsic::x86_sse3_monitor:
5129 case Intrinsic::x86_monitorx:
5130 case Intrinsic::x86_clzero: {
5131 bool Use64BitPtr =
Node->getOperand(2).getValueType() == MVT::i64;
5136 case Intrinsic::x86_sse3_monitor:
5137 if (!Subtarget->hasSSE3())
5139 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5141 case Intrinsic::x86_monitorx:
5142 if (!Subtarget->hasMWAITX())
5144 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5146 case Intrinsic::x86_clzero:
5147 if (!Subtarget->hasCLZERO())
5149 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5154 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5155 SDValue Chain = CurDAG->getCopyToReg(
Node->getOperand(0), dl, PtrReg,
5159 if (IntNo == Intrinsic::x86_sse3_monitor ||
5160 IntNo == Intrinsic::x86_monitorx) {
5162 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX,
Node->getOperand(3),
5165 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX,
Node->getOperand(4),
5170 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5172 ReplaceNode(
Node, CNode);
5178 case Intrinsic::x86_tilestored64_internal: {
5179 unsigned Opc = X86::PTILESTOREDV;
5182 SDValue Scale = getI8Imm(1, dl);
5184 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5185 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5189 Node->getOperand(3),
5195 Node->getOperand(6),
5197 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5198 ReplaceNode(
Node, CNode);
5201 case Intrinsic::x86_tileloadd64:
5202 case Intrinsic::x86_tileloaddt164:
5203 case Intrinsic::x86_tilestored64: {
5204 if (!Subtarget->hasAMXTILE())
5209 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD;
break;
5210 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1;
break;
5211 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED;
break;
5214 unsigned TIndex =
Node->getConstantOperandVal(2);
5215 SDValue TReg = getI8Imm(TIndex, dl);
5217 SDValue Scale = getI8Imm(1, dl);
5219 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5220 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5223 if (Opc == X86::PTILESTORED) {
5225 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5228 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5230 ReplaceNode(
Node, CNode);
5238 if (Subtarget->isTargetNaCl())
5242 if (Subtarget->isTarget64BitILP32()) {
5247 assert(
Target.getValueType() == MVT::i32 &&
"Unexpected VT!");
5248 SDValue ZextTarget = CurDAG->getZExtOrTrunc(
Target, dl, MVT::i64);
5249 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5250 Node->getOperand(0), ZextTarget);
5252 SelectCode(ZextTarget.
getNode());
5259 ReplaceNode(
Node, getGlobalBaseReg());
5267 CurDAG->RemoveDeadNode(
Node);
5273 if (matchBitExtract(
Node))
5278 if (tryShiftAmountMod(
Node))
5283 uint8_t
Imm =
Node->getConstantOperandVal(3);
5285 Node->getOperand(1),
Node->getOperand(2), Imm))
5291 if (tryVPTERNLOG(
Node))
5301 tryVPTESTM(
Node, N0, N1))
5304 tryVPTESTM(
Node, N1, N0))
5310 CurDAG->RemoveDeadNode(
Node);
5313 if (matchBitExtract(
Node))
5321 if (tryShrinkShlLogicImm(
Node))
5325 if (tryVPTERNLOG(
Node))
5340 if (!CurDAG->shouldOptForSize())
5344 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5350 auto *Cst = dyn_cast<ConstantSDNode>(N1);
5354 int64_t Val = Cst->getSExtValue();
5358 if (!isInt<8>(Val) && !isInt<32>(Val))
5362 if (Opcode ==
ISD::ADD && (Val == 1 || Val == -1))
5366 if (!shouldAvoidImmediateInstFormsForSize(N1.
getNode()))
5370 unsigned ROpc, MOpc;
5479 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5480 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5482 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5483 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5487 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()});
5489 CurDAG->RemoveDeadNode(
Node);
5494 CurDAG->SelectNodeTo(
Node, ROpc, NVT, MVT::i32, N0, N1);
5507 unsigned LoReg, ROpc, MOpc;
5512 ROpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5513 MOpc = Opcode ==
X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5532 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5533 bool FoldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5536 FoldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5541 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5550 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5552 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5556 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5561 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5567 VTs = CurDAG->getVTList(NVT, MVT::i32);
5569 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5571 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5576 CurDAG->RemoveDeadNode(
Node);
5586 unsigned LoReg, HiReg;
5588 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5593 Opc = UseMULXHi ? X86::MULX32Hrr
5595 : IsSigned ?
X86::IMUL32r
5597 MOpc = UseMULXHi ? X86::MULX32Hrm
5599 : IsSigned ?
X86::IMUL32m
5601 LoReg = UseMULX ? X86::EDX : X86::EAX;
5605 Opc = UseMULXHi ? X86::MULX64Hrr
5607 : IsSigned ?
X86::IMUL64r
5609 MOpc = UseMULXHi ? X86::MULX64Hrm
5611 : IsSigned ?
X86::IMUL64m
5613 LoReg = UseMULX ? X86::RDX : X86::RAX;
5618 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5619 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5622 foldedLoad = tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5627 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5636 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5637 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5640 }
else if (UseMULX) {
5641 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5642 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5647 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5648 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5654 ReplaceUses(N1.
getValue(1), Chain);
5656 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5658 SDValue Ops[] = { N1, InGlue };
5660 SDVTList VTs = CurDAG->getVTList(NVT);
5661 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5663 }
else if (UseMULX) {
5664 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5665 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5669 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5670 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5678 assert(LoReg &&
"Register for low half is not defined!");
5679 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5690 assert(HiReg &&
"Register for high half is not defined!");
5691 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5700 CurDAG->RemoveDeadNode(
Node);
5709 unsigned ROpc, MOpc;
5714 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m;
break;
5715 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m;
break;
5716 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m;
break;
5717 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m;
break;
5722 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m;
break;
5723 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m;
break;
5724 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m;
break;
5725 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m;
break;
5729 unsigned LoReg, HiReg, ClrReg;
5730 unsigned SExtOpcode;
5734 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5738 LoReg = X86::AX; HiReg = X86::DX;
5740 SExtOpcode = X86::CWD;
5743 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5744 SExtOpcode = X86::CDQ;
5747 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5748 SExtOpcode = X86::CQO;
5752 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5753 bool foldedLoad = tryFoldLoad(
Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5754 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5757 if (NVT == MVT::i8) {
5760 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5762 if (tryFoldLoad(
Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5764 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5766 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5768 ReplaceUses(N0.
getValue(1), Chain);
5770 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
5772 unsigned Opc = (
isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5774 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5775 Chain = CurDAG->getEntryNode();
5777 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX,
SDValue(Move, 0),
5782 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5783 LoReg, N0,
SDValue()).getValue(1);
5787 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5790 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5792 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0);
5796 SDValue(CurDAG->getMachineNode(
5797 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5798 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5806 SDValue(CurDAG->getMachineNode(
5807 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5808 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5809 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5817 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5818 ClrNode, InGlue).getValue(1);
5826 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5831 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
5834 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5844 if (HiReg == X86::AH && !
SDValue(
Node, 1).use_empty()) {
5845 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5846 unsigned AHExtOpcode =
5847 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5849 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
5850 MVT::Glue, AHCopy, InGlue);
5855 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
5863 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5864 LoReg, NVT, InGlue);
5865 InGlue =
Result.getValue(2);
5872 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5873 HiReg, NVT, InGlue);
5874 InGlue =
Result.getValue(2);
5879 CurDAG->RemoveDeadNode(
Node);
5888 SDValue N0 =
Node->getOperand(IsStrictCmp ? 1 : 0);
5889 SDValue N1 =
Node->getOperand(IsStrictCmp ? 2 : 1);
5895 if (Subtarget->canUseCMOV())
5904 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
5907 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
5910 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
5915 IsStrictCmp ?
Node->getOperand(0) : CurDAG->getEntryNode();
5918 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5919 Chain =
SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5922 Glue =
SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
5927 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
5931 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
5935 assert(Subtarget->canUseLAHFSAHF() &&
5936 "Target doesn't support SAHF or FCOMI?");
5937 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract,
SDValue());
5940 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.
getValue(1)), 0);
5946 CurDAG->RemoveDeadNode(
Node);
5967 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
5970 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
5972 CurDAG->RemoveDeadNode(
Node);
5986 auto *MaskC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
6004 unsigned TestOpcode;
6012 if (LeadingZeros == 0 && SavesBytes) {
6017 ShiftAmt = TrailingZeros;
6019 TestOpcode = X86::TEST64rr;
6020 }
else if (TrailingZeros == 0 && SavesBytes) {
6025 ShiftAmt = LeadingZeros;
6027 TestOpcode = X86::TEST64rr;
6028 }
else if (MaskC->hasOneUse() && !isInt<32>(Mask)) {
6031 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6032 if (PopCount == 8) {
6034 ShiftAmt = TrailingZeros;
6035 SubRegIdx = X86::sub_8bit;
6037 TestOpcode = X86::TEST8rr;
6038 }
else if (PopCount == 16) {
6040 ShiftAmt = TrailingZeros;
6041 SubRegIdx = X86::sub_16bit;
6042 SubRegVT = MVT::i16;
6043 TestOpcode = X86::TEST16rr;
6044 }
else if (PopCount == 32) {
6046 ShiftAmt = TrailingZeros;
6047 SubRegIdx = X86::sub_32bit;
6048 SubRegVT = MVT::i32;
6049 TestOpcode = X86::TEST32rr;
6053 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6055 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6058 if (SubRegIdx != 0) {
6060 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6063 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6071 unsigned ROpc, MOpc;
6078 if (isUInt<8>(Mask) &&
6079 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6083 SubRegOp = X86::sub_8bit;
6084 ROpc = X86::TEST8ri;
6085 MOpc = X86::TEST8mi;
6086 }
else if (OptForMinSize && isUInt<16>(Mask) &&
6087 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6094 SubRegOp = X86::sub_16bit;
6095 ROpc = X86::TEST16ri;
6096 MOpc = X86::TEST16mi;
6097 }
else if (isUInt<32>(Mask) && N0.
getValueType() != MVT::i16 &&
6098 ((!(Mask & 0x80000000) &&
6101 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6102 CmpVT == MVT::i32 ||
6110 SubRegOp = X86::sub_32bit;
6111 ROpc = X86::TEST32ri;
6112 MOpc = X86::TEST32mi;
6118 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6123 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6124 if (tryFoldLoad(
Node, N0.
getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6126 if (!LoadN->isSimple()) {
6128 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6129 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6130 (MOpc == X86::TEST32mi && NumVolBits != 32))
6134 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
Imm,
6135 Reg.getOperand(0) };
6136 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6138 ReplaceUses(
Reg.getValue(1),
SDValue(NewNode, 1));
6140 CurDAG->setNodeMemRefs(NewNode,
6141 {cast<LoadSDNode>(Reg)->getMemOperand()});
6145 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6147 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6150 ReplaceNode(
Node, NewNode);
6156 if (!Subtarget->hasSSE42())
6162 bool MayFoldLoad = !NeedIndex || !NeedMask;
6167 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6169 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6170 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node);
6173 if (NeedIndex || !NeedMask) {
6175 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6177 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6178 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node);
6184 CurDAG->RemoveDeadNode(
Node);
6188 if (!Subtarget->hasSSE42())
6192 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6193 Node->getOperand(1),
6195 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6196 Node->getOperand(3), InGlue).getValue(1);
6201 bool MayFoldLoad = !NeedIndex || !NeedMask;
6206 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6208 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6210 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8,
Node, InGlue);
6213 if (NeedIndex || !NeedMask) {
6215 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6217 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6218 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32,
Node, InGlue);
6223 CurDAG->RemoveDeadNode(
Node);
6235 if (foldLoadStoreIntoMemOperand(
Node))
6240 MVT VT =
Node->getSimpleValueType(0);
6242 if (Subtarget->hasSBBDepBreaking()) {
6247 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6252 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6253 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6255 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.
getValue(1)),
6264 if (VT == MVT::i8 || VT == MVT::i16) {
6265 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6266 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6270 CurDAG->RemoveDeadNode(
Node);
6284 MVT VT =
Node->getSimpleValueType(0);
6285 if (VT == MVT::i8 || VT == MVT::i16) {
6286 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6287 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6292 CurDAG->RemoveDeadNode(
Node);
6298 auto *Mgt = cast<X86MaskedGatherSDNode>(
Node);
6299 SDValue IndexOp = Mgt->getIndex();
6302 MVT ValueVT =
Node->getSimpleValueType(0);
6303 MVT MaskVT =
Mask.getSimpleValueType();
6320 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6321 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6322 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6323 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6324 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6325 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6326 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6327 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6328 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6329 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6330 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6331 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6332 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6333 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6334 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6335 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6336 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6337 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6338 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6339 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6340 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6341 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6342 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6343 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6345 assert(
EVT(MaskVT) ==
EVT(ValueVT).changeVectorElementTypeToInteger() &&
6346 "Unexpected mask VT!");
6347 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6348 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6349 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6350 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6351 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6352 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6353 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6354 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6355 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6356 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6357 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6358 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6359 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6360 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6361 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6362 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6369 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6373 SDValue PassThru = Mgt->getPassThru();
6374 SDValue Chain = Mgt->getChain();
6376 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6381 Index, Disp, Segment, Chain};
6382 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6385 Disp, Segment,
Mask, Chain};
6386 NewNode = CurDAG->getMachineNode(Opc,
SDLoc(dl), VTs, Ops);
6388 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6391 CurDAG->RemoveDeadNode(
Node);
6395 auto *Sc = cast<X86MaskedScatterSDNode>(
Node);
6397 SDValue IndexOp = Sc->getIndex();
6399 MVT ValueVT =
Value.getSimpleValueType();
6414 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6415 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6416 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6417 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6418 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6419 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6420 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6421 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6422 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6423 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6424 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6425 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6426 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6427 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6428 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6429 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6430 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6431 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6432 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6433 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6434 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6435 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6436 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6437 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6442 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6447 SDValue Chain = Sc->getChain();
6449 SDVTList VTs = CurDAG->getVTList(
Mask.getValueType(), MVT::Other);
6453 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6455 CurDAG->RemoveDeadNode(
Node);
6461 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6463 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6465 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6467 CurDAG->RemoveDeadNode(
Node);
6473 cast<SrcValueSDNode>(
Node->getOperand(1))->getValue());
6475 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6478 Ops[0] = CallIdValue;
6482 TargetOpcode::PREALLOCATED_ARG, dl,
6483 CurDAG->getVTList(TLI->
getPointerTy(CurDAG->getDataLayout()),
6488 CurDAG->RemoveDeadNode(
Node);
6495 if (!Subtarget->hasWIDEKL())
6499 switch (
Node->getOpcode()) {
6503 Opcode = X86::AESENCWIDE128KL;
6506 Opcode = X86::AESDECWIDE128KL;
6509 Opcode = X86::AESENCWIDE256KL;
6512 Opcode = X86::AESDECWIDE256KL;
6523 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0,
Node->getOperand(2),
6525 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1,
Node->getOperand(3),
6527 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2,
Node->getOperand(4),
6529 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3,
Node->getOperand(5),
6531 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4,
Node->getOperand(6),
6533 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5,
Node->getOperand(7),
6535 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6,
Node->getOperand(8),
6537 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7,
Node->getOperand(9),
6541 Opcode, dl,
Node->getVTList(),
6542 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6543 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(
Node)->getMemOperand());
6544 ReplaceNode(
Node, Res);
6552bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6554 std::vector<SDValue> &OutOps) {
6555 SDValue Op0, Op1, Op2, Op3, Op4;
6556 switch (ConstraintID) {
6559 case InlineAsm::ConstraintCode::o:
6560 case InlineAsm::ConstraintCode::v:
6561 case InlineAsm::ConstraintCode::m:
6562 case InlineAsm::ConstraintCode::X:
6563 case InlineAsm::ConstraintCode::p:
6564 if (!selectAddr(
nullptr,
Op, Op0, Op1, Op2, Op3, Op4))
6569 OutOps.push_back(Op0);
6570 OutOps.push_back(Op1);
6571 OutOps.push_back(Op2);
6572 OutOps.push_back(Op3);
6573 OutOps.push_back(Op4);
6581 return new X86DAGToDAGISel(
TM, OptLevel);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to mod...
#define GET_EGPR_IF_ENABLED(OPC)
static bool needBWI(MVT VT)
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool mayUseCarryFlag(X86::CondCode CC)
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
cl::opt< bool > IndirectBranchTracking
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N)
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
static bool isDispSafeForFrameIndex(int64_t Val)
static bool isEndbrImm64(uint64_t Imm)
#define GET_ND_IF_ENABLED(OPC)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
The address of a basic block.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Module * getModule() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
static int getUninvalidatedNodeId(SDNode *N)
virtual void emitFunctionEntryCode()
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static Type * getVoidTy(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void dump() const
Support for debugging, callable in GDB: V->dump()
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BRIND
BRIND - Indirect branch.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
CodeGenOptLevel
Code generation optimization level.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.