LLVM 19.0.0git
RISCVInstructionSelector.cpp
Go to the documentation of this file.
1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46 static const char *getName() { return DEBUG_TYPE; }
47
48private:
50 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
51
52 bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
53 bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
54
55 // tblgen-erated 'select' implementation, used as the initial selector for
56 // the patterns that don't require complex C++.
57 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
58
59 // A lowering phase that runs before any selection attempts.
60 // Returns true if the instruction was modified.
61 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
63
64 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
66
67 // Custom selection methods
69 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
71 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
72 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
73 MachineRegisterInfo &MRI, bool IsLocal = true,
74 bool IsExternWeak = false) const;
75 bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
76 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
78 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
80 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
81 MachineIRBuilder &MIB) const;
86
87 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
88 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
89
90 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
91 template <unsigned ShAmt>
92 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
93 return selectSHXADDOp(Root, ShAmt);
94 }
95
96 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
97 unsigned ShAmt) const;
98 template <unsigned ShAmt>
99 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
100 return selectSHXADD_UWOp(Root, ShAmt);
101 }
102
103 // Custom renderers for tablegen
104 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
105 int OpIdx) const;
106 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
107 int OpIdx) const;
108 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
109 int OpIdx) const;
110 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
111 int OpIdx) const;
112 void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
113 int OpIdx) const;
114
115 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
116 int OpIdx) const;
117
118 const RISCVSubtarget &STI;
119 const RISCVInstrInfo &TII;
120 const RISCVRegisterInfo &TRI;
121 const RISCVRegisterBankInfo &RBI;
122 const RISCVTargetMachine &TM;
123
124 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
125 // uses "STI." in the code generated by TableGen. We need to unify the name of
126 // Subtarget variable.
127 const RISCVSubtarget *Subtarget = &STI;
128
129#define GET_GLOBALISEL_PREDICATES_DECL
130#include "RISCVGenGlobalISel.inc"
131#undef GET_GLOBALISEL_PREDICATES_DECL
132
133#define GET_GLOBALISEL_TEMPORARIES_DECL
134#include "RISCVGenGlobalISel.inc"
135#undef GET_GLOBALISEL_TEMPORARIES_DECL
136};
137
138} // end anonymous namespace
139
140#define GET_GLOBALISEL_IMPL
141#include "RISCVGenGlobalISel.inc"
142#undef GET_GLOBALISEL_IMPL
143
144RISCVInstructionSelector::RISCVInstructionSelector(
145 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
146 const RISCVRegisterBankInfo &RBI)
147 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
148 TM(TM),
149
151#include "RISCVGenGlobalISel.inc"
154#include "RISCVGenGlobalISel.inc"
156{
157}
158
160RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
161 if (!Root.isReg())
162 return std::nullopt;
163
164 using namespace llvm::MIPatternMatch;
165 MachineRegisterInfo &MRI = MF->getRegInfo();
166
167 Register RootReg = Root.getReg();
168 Register ShAmtReg = RootReg;
169 const LLT ShiftLLT = MRI.getType(RootReg);
170 unsigned ShiftWidth = ShiftLLT.getSizeInBits();
171 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
172 // Peek through zext.
173 Register ZExtSrcReg;
174 if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
175 ShAmtReg = ZExtSrcReg;
176 }
177
178 APInt AndMask;
179 Register AndSrcReg;
180 // Try to combine the following pattern (applicable to other shift
181 // instructions as well as 32-bit ones):
182 //
183 // %4:gprb(s64) = G_AND %3, %2
184 // %5:gprb(s64) = G_LSHR %1, %4(s64)
185 //
186 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
187 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
188 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
189 // then it can be eliminated. Given register rs1 or rs2 holding a constant
190 // (the and mask), there are two cases G_AND can be erased:
191 //
192 // 1. the lowest log2(XLEN) bits of the and mask are all set
193 // 2. the bits of the register being masked are already unset (zero set)
194 if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
195 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
196 if (ShMask.isSubsetOf(AndMask)) {
197 ShAmtReg = AndSrcReg;
198 } else {
199 // SimplifyDemandedBits may have optimized the mask so try restoring any
200 // bits that are known zero.
201 KnownBits Known = KB->getKnownBits(AndSrcReg);
202 if (ShMask.isSubsetOf(AndMask | Known.Zero))
203 ShAmtReg = AndSrcReg;
204 }
205 }
206
207 APInt Imm;
209 if (mi_match(ShAmtReg, MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
210 if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
211 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
212 // to avoid the ADD.
213 ShAmtReg = Reg;
214 } else if (mi_match(ShAmtReg, MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
215 if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
216 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
217 // to generate a NEG instead of a SUB of a constant.
218 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
219 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
220 return {{[=](MachineInstrBuilder &MIB) {
221 MachineIRBuilder(*MIB.getInstr())
222 .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
223 MIB.addReg(ShAmtReg);
224 }}};
225 }
226 if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
227 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
228 // to generate a NOT instead of a SUB of a constant.
229 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
230 return {{[=](MachineInstrBuilder &MIB) {
231 MachineIRBuilder(*MIB.getInstr())
232 .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
233 .addImm(-1);
234 MIB.addReg(ShAmtReg);
235 }}};
236 }
237 }
238
239 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
240}
241
243RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
244 unsigned ShAmt) const {
245 using namespace llvm::MIPatternMatch;
246 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
248
249 if (!Root.isReg())
250 return std::nullopt;
251 Register RootReg = Root.getReg();
252
253 const unsigned XLen = STI.getXLen();
254 APInt Mask, C2;
255 Register RegY;
256 std::optional<bool> LeftShift;
257 // (and (shl y, c2), mask)
258 if (mi_match(RootReg, MRI,
259 m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
260 LeftShift = true;
261 // (and (lshr y, c2), mask)
262 else if (mi_match(RootReg, MRI,
263 m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
264 LeftShift = false;
265
266 if (LeftShift.has_value()) {
267 if (*LeftShift)
268 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
269 else
270 Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
271
272 if (Mask.isShiftedMask()) {
273 unsigned Leading = XLen - Mask.getActiveBits();
274 unsigned Trailing = Mask.countr_zero();
275 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
276 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
277 if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
278 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
279 return {{[=](MachineInstrBuilder &MIB) {
280 MachineIRBuilder(*MIB.getInstr())
281 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
282 .addImm(Trailing - C2.getLimitedValue());
283 MIB.addReg(DstReg);
284 }}};
285 }
286
287 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
288 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
289 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
290 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
291 return {{[=](MachineInstrBuilder &MIB) {
292 MachineIRBuilder(*MIB.getInstr())
293 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
294 .addImm(Leading + Trailing);
295 MIB.addReg(DstReg);
296 }}};
297 }
298 }
299 }
300
301 LeftShift.reset();
302
303 // (shl (and y, mask), c2)
304 if (mi_match(RootReg, MRI,
305 m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
306 m_ICst(C2))))
307 LeftShift = true;
308 // (lshr (and y, mask), c2)
309 else if (mi_match(RootReg, MRI,
311 m_ICst(C2))))
312 LeftShift = false;
313
314 if (LeftShift.has_value() && Mask.isShiftedMask()) {
315 unsigned Leading = XLen - Mask.getActiveBits();
316 unsigned Trailing = Mask.countr_zero();
317
318 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
319 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
320 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
321 (Trailing + C2.getLimitedValue()) == ShAmt;
322 if (!Cond)
323 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
324 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
325 Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
326 (Trailing - C2.getLimitedValue()) == ShAmt;
327
328 if (Cond) {
329 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
330 return {{[=](MachineInstrBuilder &MIB) {
331 MachineIRBuilder(*MIB.getInstr())
332 .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
333 .addImm(Trailing);
334 MIB.addReg(DstReg);
335 }}};
336 }
337 }
338
339 return std::nullopt;
340}
341
343RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
344 unsigned ShAmt) const {
345 using namespace llvm::MIPatternMatch;
346 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
348
349 if (!Root.isReg())
350 return std::nullopt;
351 Register RootReg = Root.getReg();
352
353 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
354 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
355 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
356 APInt Mask, C2;
357 Register RegX;
358 if (mi_match(
359 RootReg, MRI,
361 m_ICst(Mask))))) {
362 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
363
364 if (Mask.isShiftedMask()) {
365 unsigned Leading = Mask.countl_zero();
366 unsigned Trailing = Mask.countr_zero();
367 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
368 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
369 return {{[=](MachineInstrBuilder &MIB) {
370 MachineIRBuilder(*MIB.getInstr())
371 .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
372 .addImm(C2.getLimitedValue() - ShAmt);
373 MIB.addReg(DstReg);
374 }}};
375 }
376 }
377 }
378
379 return std::nullopt;
380}
381
383RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
384 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
386
387 if (!Root.isReg())
388 return std::nullopt;
389
390 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
391 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
392 return {{
393 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
394 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
395 }};
396 }
397
398 if (isBaseWithConstantOffset(Root, MRI)) {
399 MachineOperand &LHS = RootDef->getOperand(1);
400 MachineOperand &RHS = RootDef->getOperand(2);
401 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
402 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
403
404 int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
405 if (isInt<12>(RHSC)) {
406 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
407 return {{
408 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
409 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
410 }};
411
412 return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
413 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
414 }
415 }
416
417 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
418 // the combiner?
419 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
420 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
421}
422
423/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
424/// CC Must be an ICMP Predicate.
425static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
426 switch (CC) {
427 default:
428 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
429 case CmpInst::Predicate::ICMP_EQ:
430 return RISCVCC::COND_EQ;
431 case CmpInst::Predicate::ICMP_NE:
432 return RISCVCC::COND_NE;
433 case CmpInst::Predicate::ICMP_ULT:
434 return RISCVCC::COND_LTU;
435 case CmpInst::Predicate::ICMP_SLT:
436 return RISCVCC::COND_LT;
437 case CmpInst::Predicate::ICMP_UGE:
438 return RISCVCC::COND_GEU;
439 case CmpInst::Predicate::ICMP_SGE:
440 return RISCVCC::COND_GE;
441 }
442}
443
446 Register &RHS) {
447 // Try to fold an ICmp. If that fails, use a NE compare with X0.
449 if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
450 LHS = CondReg;
451 RHS = RISCV::X0;
453 return;
454 }
455
456 // We found an ICmp, do some canonicalizations.
457
458 // Adjust comparisons to use comparison with 0 if possible.
460 switch (Pred) {
461 case CmpInst::Predicate::ICMP_SGT:
462 // Convert X > -1 to X >= 0
463 if (*Constant == -1) {
465 RHS = RISCV::X0;
466 return;
467 }
468 break;
469 case CmpInst::Predicate::ICMP_SLT:
470 // Convert X < 1 to 0 >= X
471 if (*Constant == 1) {
473 RHS = LHS;
474 LHS = RISCV::X0;
475 return;
476 }
477 break;
478 default:
479 break;
480 }
481 }
482
483 switch (Pred) {
484 default:
485 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
486 case CmpInst::Predicate::ICMP_EQ:
487 case CmpInst::Predicate::ICMP_NE:
488 case CmpInst::Predicate::ICMP_ULT:
489 case CmpInst::Predicate::ICMP_SLT:
490 case CmpInst::Predicate::ICMP_UGE:
491 case CmpInst::Predicate::ICMP_SGE:
492 // These CCs are supported directly by RISC-V branches.
493 break;
494 case CmpInst::Predicate::ICMP_SGT:
495 case CmpInst::Predicate::ICMP_SLE:
496 case CmpInst::Predicate::ICMP_UGT:
497 case CmpInst::Predicate::ICMP_ULE:
498 // These CCs are not supported directly by RISC-V branches, but changing the
499 // direction of the CC and swapping LHS and RHS are.
500 Pred = CmpInst::getSwappedPredicate(Pred);
501 std::swap(LHS, RHS);
502 break;
503 }
504
505 CC = getRISCVCCFromICmp(Pred);
506 return;
507}
508
509bool RISCVInstructionSelector::select(MachineInstr &MI) {
510 MachineBasicBlock &MBB = *MI.getParent();
513 MachineIRBuilder MIB(MI);
514
515 preISelLower(MI, MIB, MRI);
516 const unsigned Opc = MI.getOpcode();
517
518 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
519 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
520 const Register DefReg = MI.getOperand(0).getReg();
521 const LLT DefTy = MRI.getType(DefReg);
522
523 const RegClassOrRegBank &RegClassOrBank =
524 MRI.getRegClassOrRegBank(DefReg);
525
526 const TargetRegisterClass *DefRC =
527 RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
528 if (!DefRC) {
529 if (!DefTy.isValid()) {
530 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
531 return false;
532 }
533
534 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
535 DefRC = getRegClassForTypeOnBank(DefTy, RB);
536 if (!DefRC) {
537 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
538 return false;
539 }
540 }
541
542 MI.setDesc(TII.get(TargetOpcode::PHI));
543 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
544 }
545
546 // Certain non-generic instructions also need some special handling.
547 if (MI.isCopy())
548 return selectCopy(MI, MRI);
549
550 return true;
551 }
552
553 if (selectImpl(MI, *CoverageInfo))
554 return true;
555
556 switch (Opc) {
557 case TargetOpcode::G_ANYEXT:
558 case TargetOpcode::G_PTRTOINT:
559 case TargetOpcode::G_INTTOPTR:
560 case TargetOpcode::G_TRUNC:
561 case TargetOpcode::G_FREEZE:
562 return selectCopy(MI, MRI);
563 case TargetOpcode::G_CONSTANT: {
564 Register DstReg = MI.getOperand(0).getReg();
565 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
566
567 if (!materializeImm(DstReg, Imm, MIB))
568 return false;
569
570 MI.eraseFromParent();
571 return true;
572 }
573 case TargetOpcode::G_FCONSTANT: {
574 // TODO: Use constant pool for complext constants.
575 // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
576 Register DstReg = MI.getOperand(0).getReg();
577 const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
578 APInt Imm = FPimm.bitcastToAPInt();
579 unsigned Size = MRI.getType(DstReg).getSizeInBits();
580 if (Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
581 Register GPRReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
582 if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
583 return false;
584
585 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X : RISCV::FMV_W_X;
586 auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
587 if (!FMV.constrainAllUses(TII, TRI, RBI))
588 return false;
589 } else {
590 assert(Size == 64 && !Subtarget->is64Bit() &&
591 "Unexpected size or subtarget");
592 // Split into two pieces and build through the stack.
593 Register GPRRegHigh = MRI.createVirtualRegister(&RISCV::GPRRegClass);
594 Register GPRRegLow = MRI.createVirtualRegister(&RISCV::GPRRegClass);
595 if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
596 MIB))
597 return false;
598 if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
599 return false;
600 MachineInstrBuilder PairF64 = MIB.buildInstr(
601 RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
602 if (!PairF64.constrainAllUses(TII, TRI, RBI))
603 return false;
604 }
605
606 MI.eraseFromParent();
607 return true;
608 }
609 case TargetOpcode::G_GLOBAL_VALUE: {
610 auto *GV = MI.getOperand(1).getGlobal();
611 if (GV->isThreadLocal()) {
612 // TODO: implement this case.
613 return false;
614 }
615
616 return selectAddr(MI, MIB, MRI, GV->isDSOLocal(),
617 GV->hasExternalWeakLinkage());
618 }
619 case TargetOpcode::G_JUMP_TABLE:
620 case TargetOpcode::G_CONSTANT_POOL:
621 return selectAddr(MI, MIB, MRI);
622 case TargetOpcode::G_BRCOND: {
625 getOperandsForBranch(MI.getOperand(0).getReg(), MRI, CC, LHS, RHS);
626
627 auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
628 .addMBB(MI.getOperand(1).getMBB());
629 MI.eraseFromParent();
630 return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
631 }
632 case TargetOpcode::G_BRJT: {
633 // FIXME: Move to legalization?
634 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
635 unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());
636 assert((EntrySize == 4 || (Subtarget->is64Bit() && EntrySize == 8)) &&
637 "Unsupported jump-table entry size");
638 assert(
642 "Unexpected jump-table entry kind");
643
644 auto SLL =
645 MIB.buildInstr(RISCV::SLLI, {&RISCV::GPRRegClass}, {MI.getOperand(2)})
646 .addImm(Log2_32(EntrySize));
647 if (!SLL.constrainAllUses(TII, TRI, RBI))
648 return false;
649
650 // TODO: Use SHXADD. Moving to legalization would fix this automatically.
651 auto ADD = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
652 {MI.getOperand(0), SLL.getReg(0)});
653 if (!ADD.constrainAllUses(TII, TRI, RBI))
654 return false;
655
656 unsigned LdOpc = EntrySize == 8 ? RISCV::LD : RISCV::LW;
657 auto Dest =
658 MIB.buildInstr(LdOpc, {&RISCV::GPRRegClass}, {ADD.getReg(0)})
659 .addImm(0)
660 .addMemOperand(MF.getMachineMemOperand(
662 EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout()))));
663 if (!Dest.constrainAllUses(TII, TRI, RBI))
664 return false;
665
666 // If the Kind is EK_LabelDifference32, the table stores an offset from
667 // the location of the table. Add the table address to get an absolute
668 // address.
670 Dest = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
671 {Dest.getReg(0), MI.getOperand(0)});
672 if (!Dest.constrainAllUses(TII, TRI, RBI))
673 return false;
674 }
675
676 auto Branch =
677 MIB.buildInstr(RISCV::PseudoBRIND, {}, {Dest.getReg(0)}).addImm(0);
678 if (!Branch.constrainAllUses(TII, TRI, RBI))
679 return false;
680
681 MI.eraseFromParent();
682 return true;
683 }
684 case TargetOpcode::G_BRINDIRECT:
685 MI.setDesc(TII.get(RISCV::PseudoBRIND));
686 MI.addOperand(MachineOperand::CreateImm(0));
688 case TargetOpcode::G_SEXT_INREG:
689 return selectSExtInreg(MI, MIB);
690 case TargetOpcode::G_FRAME_INDEX: {
691 // TODO: We may want to replace this code with the SelectionDAG patterns,
692 // which fail to get imported because it uses FrameAddrRegImm, which is a
693 // ComplexPattern
694 MI.setDesc(TII.get(RISCV::ADDI));
695 MI.addOperand(MachineOperand::CreateImm(0));
697 }
698 case TargetOpcode::G_SELECT:
699 return selectSelect(MI, MIB, MRI);
700 case TargetOpcode::G_FCMP:
701 return selectFPCompare(MI, MIB, MRI);
702 case TargetOpcode::G_FENCE: {
703 AtomicOrdering FenceOrdering =
704 static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
705 SyncScope::ID FenceSSID =
706 static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
707 emitFence(FenceOrdering, FenceSSID, MIB);
708 MI.eraseFromParent();
709 return true;
710 }
711 case TargetOpcode::G_IMPLICIT_DEF:
712 return selectImplicitDef(MI, MIB, MRI);
713 case TargetOpcode::G_MERGE_VALUES:
714 return selectMergeValues(MI, MIB, MRI);
715 case TargetOpcode::G_UNMERGE_VALUES:
716 return selectUnmergeValues(MI, MIB, MRI);
717 default:
718 return false;
719 }
720}
721
722bool RISCVInstructionSelector::selectMergeValues(
724 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
725
726 // Build a F64 Pair from operands
727 if (MI.getNumOperands() != 3)
728 return false;
729 Register Dst = MI.getOperand(0).getReg();
730 Register Lo = MI.getOperand(1).getReg();
731 Register Hi = MI.getOperand(2).getReg();
732 if (!isRegInFprb(Dst, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
733 return false;
734 MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
736}
737
738bool RISCVInstructionSelector::selectUnmergeValues(
740 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
741
742 // Split F64 Src into two s32 parts
743 if (MI.getNumOperands() != 3)
744 return false;
745 Register Src = MI.getOperand(2).getReg();
746 Register Lo = MI.getOperand(0).getReg();
747 Register Hi = MI.getOperand(1).getReg();
748 if (!isRegInFprb(Src, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
749 return false;
750 MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
752}
753
754bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
755 MachineIRBuilder &MIB,
757 Register PtrReg = Op.getReg();
758 assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
759
760 const LLT sXLen = LLT::scalar(STI.getXLen());
761 auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
762 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
763 Op.setReg(PtrToInt.getReg(0));
764 return select(*PtrToInt);
765}
766
767void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
768 MachineIRBuilder &MIB,
770 switch (MI.getOpcode()) {
771 case TargetOpcode::G_PTR_ADD: {
772 Register DstReg = MI.getOperand(0).getReg();
773 const LLT sXLen = LLT::scalar(STI.getXLen());
774
775 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
776 MI.setDesc(TII.get(TargetOpcode::G_ADD));
777 MRI.setType(DstReg, sXLen);
778 break;
779 }
780 case TargetOpcode::G_PTRMASK: {
781 Register DstReg = MI.getOperand(0).getReg();
782 const LLT sXLen = LLT::scalar(STI.getXLen());
783 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
784 MI.setDesc(TII.get(TargetOpcode::G_AND));
785 MRI.setType(DstReg, sXLen);
786 }
787 }
788}
789
790void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
791 const MachineInstr &MI,
792 int OpIdx) const {
793 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
794 "Expected G_CONSTANT");
795 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
796 MIB.addImm(-CstVal);
797}
798
799void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
800 const MachineInstr &MI,
801 int OpIdx) const {
802 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
803 "Expected G_CONSTANT");
804 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
805 MIB.addImm(STI.getXLen() - CstVal);
806}
807
808void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
809 const MachineInstr &MI,
810 int OpIdx) const {
811 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
812 "Expected G_CONSTANT");
813 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
814 MIB.addImm(32 - CstVal);
815}
816
817void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
818 const MachineInstr &MI,
819 int OpIdx) const {
820 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
821 "Expected G_CONSTANT");
822 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
823 MIB.addImm(CstVal + 1);
824}
825
826void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
827 const MachineInstr &MI,
828 int OpIdx) const {
829 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
830 "Expected G_CONSTANT");
831 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
832 MIB.addImm(CstVal);
833}
834
835void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
836 const MachineInstr &MI,
837 int OpIdx) const {
838 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
839 "Expected G_CONSTANT");
840 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
842}
843
844const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
845 LLT Ty, const RegisterBank &RB) const {
846 if (RB.getID() == RISCV::GPRBRegBankID) {
847 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
848 return &RISCV::GPRRegClass;
849 }
850
851 if (RB.getID() == RISCV::FPRBRegBankID) {
852 if (Ty.getSizeInBits() == 16)
853 return &RISCV::FPR16RegClass;
854 if (Ty.getSizeInBits() == 32)
855 return &RISCV::FPR32RegClass;
856 if (Ty.getSizeInBits() == 64)
857 return &RISCV::FPR64RegClass;
858 }
859
860 if (RB.getID() == RISCV::VRBRegBankID) {
861 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
862 return &RISCV::VRRegClass;
863
864 if (Ty.getSizeInBits().getKnownMinValue() == 128)
865 return &RISCV::VRM2RegClass;
866
867 if (Ty.getSizeInBits().getKnownMinValue() == 256)
868 return &RISCV::VRM4RegClass;
869
870 if (Ty.getSizeInBits().getKnownMinValue() == 512)
871 return &RISCV::VRM8RegClass;
872 }
873
874 return nullptr;
875}
876
877bool RISCVInstructionSelector::isRegInGprb(Register Reg,
878 MachineRegisterInfo &MRI) const {
879 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
880}
881
882bool RISCVInstructionSelector::isRegInFprb(Register Reg,
883 MachineRegisterInfo &MRI) const {
884 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
885}
886
887bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
888 MachineRegisterInfo &MRI) const {
889 Register DstReg = MI.getOperand(0).getReg();
890
891 if (DstReg.isPhysical())
892 return true;
893
894 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
895 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
896 assert(DstRC &&
897 "Register class not available for LLT, register bank combination");
898
899 // No need to constrain SrcReg. It will get constrained when
900 // we hit another of its uses or its defs.
901 // Copies do not have constraints.
902 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
903 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
904 << " operand\n");
905 return false;
906 }
907
908 MI.setDesc(TII.get(RISCV::COPY));
909 return true;
910}
911
912bool RISCVInstructionSelector::selectImplicitDef(
914 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
915
916 const Register DstReg = MI.getOperand(0).getReg();
917 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
918 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
919
920 assert(DstRC &&
921 "Register class not available for LLT, register bank combination");
922
923 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
924 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
925 << " operand\n");
926 }
927 MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
928 return true;
929}
930
931bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
932 MachineIRBuilder &MIB) const {
934
935 if (Imm == 0) {
936 MIB.buildCopy(DstReg, Register(RISCV::X0));
937 RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, MRI);
938 return true;
939 }
940
942 unsigned NumInsts = Seq.size();
943 Register SrcReg = RISCV::X0;
944
945 for (unsigned i = 0; i < NumInsts; i++) {
946 Register TmpReg = i < NumInsts - 1
947 ? MRI.createVirtualRegister(&RISCV::GPRRegClass)
948 : DstReg;
949 const RISCVMatInt::Inst &I = Seq[i];
951
952 switch (I.getOpndKind()) {
953 case RISCVMatInt::Imm:
954 // clang-format off
955 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
956 .addImm(I.getImm());
957 // clang-format on
958 break;
960 Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
961 {SrcReg, Register(RISCV::X0)});
962 break;
964 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
965 break;
967 Result =
968 MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
969 break;
970 }
971
972 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
973 return false;
974
975 SrcReg = TmpReg;
976 }
977
978 return true;
979}
980
981bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
982 MachineIRBuilder &MIB,
984 bool IsLocal,
985 bool IsExternWeak) const {
986 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
987 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
988 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
989 "Unexpected opcode");
990
991 const MachineOperand &DispMO = MI.getOperand(1);
992
993 Register DefReg = MI.getOperand(0).getReg();
994 const LLT DefTy = MRI.getType(DefReg);
995
996 // When HWASAN is used and tagging of global variables is enabled
997 // they should be accessed via the GOT, since the tagged address of a global
998 // is incompatible with existing code models. This also applies to non-pic
999 // mode.
1000 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1001 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1002 // Use PC-relative addressing to access the symbol. This generates the
1003 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1004 // %pcrel_lo(auipc)).
1005 MI.setDesc(TII.get(RISCV::PseudoLLA));
1007 }
1008
1009 // Use PC-relative addressing to access the GOT for this symbol, then
1010 // load the address from the GOT. This generates the pattern (PseudoLGA
1011 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1012 // %pcrel_lo(auipc))).
1013 MachineFunction &MF = *MI.getParent()->getParent();
1018 DefTy, Align(DefTy.getSizeInBits() / 8));
1019
1020 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1021 .addDisp(DispMO, 0)
1023
1024 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1025 return false;
1026
1027 MI.eraseFromParent();
1028 return true;
1029 }
1030
1031 switch (TM.getCodeModel()) {
1032 default: {
1033 reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1034 getName(), "Unsupported code model for lowering", MI);
1035 return false;
1036 }
1037 case CodeModel::Small: {
1038 // Must lie within a single 2 GiB address range and must lie between
1039 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1040 // (lui %hi(sym)) %lo(sym)).
1041 Register AddrHiDest = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1042 MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1043 .addDisp(DispMO, 0, RISCVII::MO_HI);
1044
1045 if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1046 return false;
1047
1048 auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1049 .addDisp(DispMO, 0, RISCVII::MO_LO);
1050
1051 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1052 return false;
1053
1054 MI.eraseFromParent();
1055 return true;
1056 }
1057 case CodeModel::Medium:
1058 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1059 // relocation needs to reference a label that points to the auipc
1060 // instruction itself, not the global. This cannot be done inside the
1061 // instruction selector.
1062 if (IsExternWeak) {
1063 // An extern weak symbol may be undefined, i.e. have value 0, which may
1064 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1065 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1066 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1067 MachineFunction &MF = *MI.getParent()->getParent();
1072 DefTy, Align(DefTy.getSizeInBits() / 8));
1073
1074 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1075 .addDisp(DispMO, 0)
1077
1078 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1079 return false;
1080
1081 MI.eraseFromParent();
1082 return true;
1083 }
1084
1085 // Generate a sequence for accessing addresses within any 2GiB range
1086 // within the address space. This generates the pattern (PseudoLLA sym),
1087 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1088 MI.setDesc(TII.get(RISCV::PseudoLLA));
1090 }
1091
1092 return false;
1093}
1094
1095bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
1096 MachineIRBuilder &MIB) const {
1097 if (!STI.isRV64())
1098 return false;
1099
1100 const MachineOperand &Size = MI.getOperand(2);
1101 // Only Size == 32 (i.e. shift by 32 bits) is acceptable at this point.
1102 if (!Size.isImm() || Size.getImm() != 32)
1103 return false;
1104
1105 const MachineOperand &Src = MI.getOperand(1);
1106 const MachineOperand &Dst = MI.getOperand(0);
1107 // addiw rd, rs, 0 (i.e. sext.w rd, rs)
1108 MachineInstr *NewMI =
1109 MIB.buildInstr(RISCV::ADDIW, {Dst.getReg()}, {Src.getReg()}).addImm(0U);
1110
1111 if (!constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI))
1112 return false;
1113
1114 MI.eraseFromParent();
1115 return true;
1116}
1117
1118bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1119 MachineIRBuilder &MIB,
1120 MachineRegisterInfo &MRI) const {
1121 auto &SelectMI = cast<GSelect>(MI);
1122
1123 Register LHS, RHS;
1125 getOperandsForBranch(SelectMI.getCondReg(), MRI, CC, LHS, RHS);
1126
1127 Register DstReg = SelectMI.getReg(0);
1128
1129 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1130 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1131 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1132 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1133 : RISCV::Select_FPR64_Using_CC_GPR;
1134 }
1135
1136 MachineInstr *Result = MIB.buildInstr(Opc)
1137 .addDef(DstReg)
1138 .addReg(LHS)
1139 .addReg(RHS)
1140 .addImm(CC)
1141 .addReg(SelectMI.getTrueReg())
1142 .addReg(SelectMI.getFalseReg());
1143 MI.eraseFromParent();
1144 return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1145}
1146
1147// Convert an FCMP predicate to one of the supported F or D instructions.
1148static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1149 assert((Size == 32 || Size == 64) && "Unsupported size");
1150 switch (Pred) {
1151 default:
1152 llvm_unreachable("Unsupported predicate");
1153 case CmpInst::FCMP_OLT:
1154 return Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1155 case CmpInst::FCMP_OLE:
1156 return Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1157 case CmpInst::FCMP_OEQ:
1158 return Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1159 }
1160}
1161
1162// Try legalizing an FCMP by swapping or inverting the predicate to one that
1163// is supported.
1165 CmpInst::Predicate &Pred, bool &NeedInvert) {
1166 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1167 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1168 Pred == CmpInst::FCMP_OEQ;
1169 };
1170
1171 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1172
1174 if (isLegalFCmpPredicate(InvPred)) {
1175 Pred = InvPred;
1176 std::swap(LHS, RHS);
1177 return true;
1178 }
1179
1180 InvPred = CmpInst::getInversePredicate(Pred);
1181 NeedInvert = true;
1182 if (isLegalFCmpPredicate(InvPred)) {
1183 Pred = InvPred;
1184 return true;
1185 }
1186 InvPred = CmpInst::getSwappedPredicate(InvPred);
1187 if (isLegalFCmpPredicate(InvPred)) {
1188 Pred = InvPred;
1189 std::swap(LHS, RHS);
1190 return true;
1191 }
1192
1193 return false;
1194}
1195
1196// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1197// the result in DstReg.
1198// FIXME: Maybe we should expand this earlier.
1199bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1200 MachineIRBuilder &MIB,
1201 MachineRegisterInfo &MRI) const {
1202 auto &CmpMI = cast<GFCmp>(MI);
1203 CmpInst::Predicate Pred = CmpMI.getCond();
1204
1205 Register DstReg = CmpMI.getReg(0);
1206 Register LHS = CmpMI.getLHSReg();
1207 Register RHS = CmpMI.getRHSReg();
1208
1209 unsigned Size = MRI.getType(LHS).getSizeInBits();
1210 assert((Size == 32 || Size == 64) && "Unexpected size");
1211
1212 Register TmpReg = DstReg;
1213
1214 bool NeedInvert = false;
1215 // First try swapping operands or inverting.
1216 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1217 if (NeedInvert)
1218 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1219 auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1220 if (!Cmp.constrainAllUses(TII, TRI, RBI))
1221 return false;
1222 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1223 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1224 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1226 {&RISCV::GPRRegClass}, {LHS, RHS});
1227 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1228 return false;
1230 {&RISCV::GPRRegClass}, {RHS, LHS});
1231 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1232 return false;
1233 if (NeedInvert)
1234 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1235 auto Or =
1236 MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1237 if (!Or.constrainAllUses(TII, TRI, RBI))
1238 return false;
1239 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1240 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1241 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1242 NeedInvert = Pred == CmpInst::FCMP_UNO;
1244 {&RISCV::GPRRegClass}, {LHS, LHS});
1245 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1246 return false;
1248 {&RISCV::GPRRegClass}, {RHS, RHS});
1249 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1250 return false;
1251 if (NeedInvert)
1252 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1253 auto And =
1254 MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1255 if (!And.constrainAllUses(TII, TRI, RBI))
1256 return false;
1257 } else
1258 llvm_unreachable("Unhandled predicate");
1259
1260 // Emit an XORI to invert the result if needed.
1261 if (NeedInvert) {
1262 auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1263 if (!Xor.constrainAllUses(TII, TRI, RBI))
1264 return false;
1265 }
1266
1267 MI.eraseFromParent();
1268 return true;
1269}
1270
1271void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1272 SyncScope::ID FenceSSID,
1273 MachineIRBuilder &MIB) const {
1274 if (STI.hasStdExtZtso()) {
1275 // The only fence that needs an instruction is a sequentially-consistent
1276 // cross-thread fence.
1277 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1278 FenceSSID == SyncScope::System) {
1279 // fence rw, rw
1280 MIB.buildInstr(RISCV::FENCE, {}, {})
1283 return;
1284 }
1285
1286 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1287 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1288 return;
1289 }
1290
1291 // singlethread fences only synchronize with signal handlers on the same
1292 // thread and thus only need to preserve instruction order, not actually
1293 // enforce memory ordering.
1294 if (FenceSSID == SyncScope::SingleThread) {
1295 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1296 return;
1297 }
1298
1299 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1300 // Manual: Volume I.
1301 unsigned Pred, Succ;
1302 switch (FenceOrdering) {
1303 default:
1304 llvm_unreachable("Unexpected ordering");
1305 case AtomicOrdering::AcquireRelease:
1306 // fence acq_rel -> fence.tso
1307 MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1308 return;
1309 case AtomicOrdering::Acquire:
1310 // fence acquire -> fence r, rw
1311 Pred = RISCVFenceField::R;
1313 break;
1314 case AtomicOrdering::Release:
1315 // fence release -> fence rw, w
1317 Succ = RISCVFenceField::W;
1318 break;
1319 case AtomicOrdering::SequentiallyConsistent:
1320 // fence seq_cst -> fence rw, rw
1323 break;
1324 }
1325 MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1326}
1327
1328namespace llvm {
1331 RISCVSubtarget &Subtarget,
1332 RISCVRegisterBankInfo &RBI) {
1333 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1334}
1335} // end namespace llvm
unsigned const MachineRegisterInfo * MRI
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
static StringRef getName(Value *V)
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI, RISCVCC::CondCode &CC, Register &LHS, Register &RHS)
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
static bool legalizeFCmpPredicate(Register &LHS, Register &RHS, CmpInst::Predicate &Pred, bool &NeedInvert)
const SmallVectorImpl< MachineOperand > & Cond
This file declares the targeting of the RegisterBankInfo class for RISC-V.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
APInt bitcastToAPInt() const
Definition: APFloat.h:1246
Class for arbitrary precision integers.
Definition: APInt.h:77
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1447
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1090
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:454
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:993
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:996
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:999
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:1001
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:1004
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:1000
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:1002
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:1003
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:1167
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:1129
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:161
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
virtual bool select(MachineInstr &I)=0
Select the (possibly generic) instruction I to only use target-specific opcodes.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isValid() const
Definition: LowLevelType.h:145
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
Helper class to build MachineInstr.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:564
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:341
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:574
unsigned getEntrySize(const DataLayout &TD) const
getEntrySize - Return the size of each entry in the jump table.
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
unsigned getEntryAlignment(const DataLayout &TD) const
getEntryAlignment - Return the alignment of each entry in the jump table.
JTEntryKind getEntryKind() const
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
T get() const
Returns the value of the specified pointer type.
Definition: PointerUnion.h:155
T dyn_cast() const
Returns the current pointer if it is of the specified pointer type, otherwise returns null.
Definition: PointerUnion.h:162
This class provides the information for the target register banks.
This class implements the register bank concept.
Definition: RegisterBank.h:28
unsigned getID() const
Get the identifier of this register bank.
Definition: RegisterBank.h:45
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
size_t size() const
Definition: SmallVector.h:91
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
unsigned getID() const
Return the register class ID number.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:240
operand_type_match m_Reg()
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
unsigned getBrCond(CondCode CC, bool Imm=false)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
@ System
Synchronized with respect to all concurrently executing threads.
Definition: LLVMContext.h:57
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:155
std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition: Utils.cpp:307
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:324
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:275
void reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, MachineOptimizationRemarkEmitter &MORE, MachineOptimizationRemarkMissed &R)
Report an ISel error as a missed optimization remark to the LLVMContext's diagnostic stream.
Definition: Utils.cpp:275
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Xor
Bitwise or logical XOR of integers.
InstructionSelector * createRISCVInstructionSelector(const RISCVTargetMachine &TM, RISCVSubtarget &Subtarget, RISCVRegisterBankInfo &RBI)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define MORE()
Definition: regcomp.c:252
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Matching combinators.
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.