LLVM 17.0.0git
MipsExpandPseudo.cpp
Go to the documentation of this file.
1//===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling, if-conversion, and other late
11// optimizations. This pass should be run after register allocation but before
12// the post-regalloc scheduling pass.
13//
14// This is currently only used for expanding atomic pseudos after register
15// allocation. We do this to avoid the fast register allocator introducing
16// spills between ll and sc. These stores cause some MIPS implementations to
17// abort the atomic RMW sequence.
18//
19//===----------------------------------------------------------------------===//
20
21#include "Mips.h"
22#include "MipsInstrInfo.h"
23#include "MipsSubtarget.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "mips-pseudo"
31
32namespace {
33 class MipsExpandPseudo : public MachineFunctionPass {
34 public:
35 static char ID;
36 MipsExpandPseudo() : MachineFunctionPass(ID) {}
37
38 const MipsInstrInfo *TII;
39 const MipsSubtarget *STI;
40
41 bool runOnMachineFunction(MachineFunction &Fn) override;
42
45 MachineFunctionProperties::Property::NoVRegs);
46 }
47
48 StringRef getPassName() const override {
49 return "Mips pseudo instruction expansion pass";
50 }
51
52 private:
53 bool expandAtomicCmpSwap(MachineBasicBlock &MBB,
56 bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB,
59
60 bool expandAtomicBinOp(MachineBasicBlock &BB,
62 MachineBasicBlock::iterator &NMBBI, unsigned Size);
63 bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
66
69 bool expandMBB(MachineBasicBlock &MBB);
70 };
71 char MipsExpandPseudo::ID = 0;
72}
73
74bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
77
78 MachineFunction *MF = BB.getParent();
79
80 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
81 DebugLoc DL = I->getDebugLoc();
82 unsigned LL, SC;
83
84 unsigned ZERO = Mips::ZERO;
85 unsigned BNE = Mips::BNE;
86 unsigned BEQ = Mips::BEQ;
87 unsigned SEOp =
88 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
89
90 if (STI->inMicroMipsMode()) {
91 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
92 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
93 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
94 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
95 } else {
96 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
97 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
98 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
99 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
100 }
101
102 Register Dest = I->getOperand(0).getReg();
103 Register Ptr = I->getOperand(1).getReg();
104 Register Mask = I->getOperand(2).getReg();
105 Register ShiftCmpVal = I->getOperand(3).getReg();
106 Register Mask2 = I->getOperand(4).getReg();
107 Register ShiftNewVal = I->getOperand(5).getReg();
108 Register ShiftAmnt = I->getOperand(6).getReg();
109 Register Scratch = I->getOperand(7).getReg();
110 Register Scratch2 = I->getOperand(8).getReg();
111
112 // insert new blocks after the current block
113 const BasicBlock *LLVM_BB = BB.getBasicBlock();
114 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
115 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
116 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
117 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
119 MF->insert(It, loop1MBB);
120 MF->insert(It, loop2MBB);
121 MF->insert(It, sinkMBB);
122 MF->insert(It, exitMBB);
123
124 // Transfer the remainder of BB and its successor edges to exitMBB.
125 exitMBB->splice(exitMBB->begin(), &BB,
126 std::next(MachineBasicBlock::iterator(I)), BB.end());
128
129 // thisMBB:
130 // ...
131 // fallthrough --> loop1MBB
133 loop1MBB->addSuccessor(sinkMBB);
134 loop1MBB->addSuccessor(loop2MBB);
135 loop1MBB->normalizeSuccProbs();
136 loop2MBB->addSuccessor(loop1MBB);
137 loop2MBB->addSuccessor(sinkMBB);
138 loop2MBB->normalizeSuccProbs();
139 sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
140
141 // loop1MBB:
142 // ll dest, 0(ptr)
143 // and Mask', dest, Mask
144 // bne Mask', ShiftCmpVal, exitMBB
145 BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
146 BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
147 .addReg(Scratch)
148 .addReg(Mask);
149 BuildMI(loop1MBB, DL, TII->get(BNE))
150 .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
151
152 // loop2MBB:
153 // and dest, dest, mask2
154 // or dest, dest, ShiftNewVal
155 // sc dest, dest, 0(ptr)
156 // beq dest, $0, loop1MBB
157 BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
158 .addReg(Scratch, RegState::Kill)
159 .addReg(Mask2);
160 BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
161 .addReg(Scratch, RegState::Kill)
162 .addReg(ShiftNewVal);
163 BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
164 .addReg(Scratch, RegState::Kill)
165 .addReg(Ptr)
166 .addImm(0);
167 BuildMI(loop2MBB, DL, TII->get(BEQ))
168 .addReg(Scratch, RegState::Kill)
169 .addReg(ZERO)
170 .addMBB(loop1MBB);
171
172 // sinkMBB:
173 // srl srlres, Mask', shiftamt
174 // sign_extend dest,srlres
175 BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
176 .addReg(Scratch2)
177 .addReg(ShiftAmnt);
178 if (STI->hasMips32r2()) {
179 BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
180 } else {
181 const unsigned ShiftImm =
182 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
183 BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
184 .addReg(Dest, RegState::Kill)
185 .addImm(ShiftImm);
186 BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
187 .addReg(Dest, RegState::Kill)
188 .addImm(ShiftImm);
189 }
190
191 LivePhysRegs LiveRegs;
192 computeAndAddLiveIns(LiveRegs, *loop1MBB);
193 computeAndAddLiveIns(LiveRegs, *loop2MBB);
194 computeAndAddLiveIns(LiveRegs, *sinkMBB);
195 computeAndAddLiveIns(LiveRegs, *exitMBB);
196
197 NMBBI = BB.end();
198 I->eraseFromParent();
199 return true;
200}
201
202bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
205
206 const unsigned Size =
207 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
208 MachineFunction *MF = BB.getParent();
209
210 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
211 DebugLoc DL = I->getDebugLoc();
212
213 unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
214
215 if (Size == 4) {
216 if (STI->inMicroMipsMode()) {
217 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
218 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
219 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
220 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
221 } else {
222 LL = STI->hasMips32r6()
223 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
224 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
225 SC = STI->hasMips32r6()
226 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
227 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
228 BNE = Mips::BNE;
229 BEQ = Mips::BEQ;
230 }
231
232 ZERO = Mips::ZERO;
233 MOVE = Mips::OR;
234 } else {
235 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
236 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
237 ZERO = Mips::ZERO_64;
238 BNE = Mips::BNE64;
239 BEQ = Mips::BEQ64;
240 MOVE = Mips::OR64;
241 }
242
243 Register Dest = I->getOperand(0).getReg();
244 Register Ptr = I->getOperand(1).getReg();
245 Register OldVal = I->getOperand(2).getReg();
246 Register NewVal = I->getOperand(3).getReg();
247 Register Scratch = I->getOperand(4).getReg();
248
249 // insert new blocks after the current block
250 const BasicBlock *LLVM_BB = BB.getBasicBlock();
251 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
252 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
253 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
255 MF->insert(It, loop1MBB);
256 MF->insert(It, loop2MBB);
257 MF->insert(It, exitMBB);
258
259 // Transfer the remainder of BB and its successor edges to exitMBB.
260 exitMBB->splice(exitMBB->begin(), &BB,
261 std::next(MachineBasicBlock::iterator(I)), BB.end());
263
264 // thisMBB:
265 // ...
266 // fallthrough --> loop1MBB
268 loop1MBB->addSuccessor(exitMBB);
269 loop1MBB->addSuccessor(loop2MBB);
270 loop1MBB->normalizeSuccProbs();
271 loop2MBB->addSuccessor(loop1MBB);
272 loop2MBB->addSuccessor(exitMBB);
273 loop2MBB->normalizeSuccProbs();
274
275 // loop1MBB:
276 // ll dest, 0(ptr)
277 // bne dest, oldval, exitMBB
278 BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
279 BuildMI(loop1MBB, DL, TII->get(BNE))
280 .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
281
282 // loop2MBB:
283 // move scratch, NewVal
284 // sc Scratch, Scratch, 0(ptr)
285 // beq Scratch, $0, loop1MBB
286 BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
287 BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
288 .addReg(Scratch).addReg(Ptr).addImm(0);
289 BuildMI(loop2MBB, DL, TII->get(BEQ))
290 .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
291
292 LivePhysRegs LiveRegs;
293 computeAndAddLiveIns(LiveRegs, *loop1MBB);
294 computeAndAddLiveIns(LiveRegs, *loop2MBB);
295 computeAndAddLiveIns(LiveRegs, *exitMBB);
296
297 NMBBI = BB.end();
298 I->eraseFromParent();
299 return true;
300}
301
302bool MipsExpandPseudo::expandAtomicBinOpSubword(
305
306 MachineFunction *MF = BB.getParent();
307
308 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
309 DebugLoc DL = I->getDebugLoc();
310
311 unsigned LL, SC, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
312 unsigned BEQ = Mips::BEQ;
313 unsigned SEOp = Mips::SEH;
314
315 if (STI->inMicroMipsMode()) {
316 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
317 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
318 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
319 SLT = Mips::SLT_MM;
320 SLTu = Mips::SLTu_MM;
321 OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
322 MOVN = Mips::MOVN_I_MM;
323 MOVZ = Mips::MOVZ_I_MM;
324 SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
325 SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
326 } else {
327 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
328 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
329 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
330 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
331 SLT = Mips::SLT;
332 SLTu = Mips::SLTu;
333 OR = Mips::OR;
334 MOVN = Mips::MOVN_I_I;
335 MOVZ = Mips::MOVZ_I_I;
336 SELNEZ = Mips::SELNEZ;
337 SELEQZ = Mips::SELEQZ;
338 }
339
340 bool IsSwap = false;
341 bool IsNand = false;
342 bool IsMin = false;
343 bool IsMax = false;
344 bool IsUnsigned = false;
345
346 unsigned Opcode = 0;
347 switch (I->getOpcode()) {
348 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
349 SEOp = Mips::SEB;
350 [[fallthrough]];
351 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
352 IsNand = true;
353 break;
354 case Mips::ATOMIC_SWAP_I8_POSTRA:
355 SEOp = Mips::SEB;
356 [[fallthrough]];
357 case Mips::ATOMIC_SWAP_I16_POSTRA:
358 IsSwap = true;
359 break;
360 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
361 SEOp = Mips::SEB;
362 [[fallthrough]];
363 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
364 Opcode = Mips::ADDu;
365 break;
366 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
367 SEOp = Mips::SEB;
368 [[fallthrough]];
369 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
370 Opcode = Mips::SUBu;
371 break;
372 case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
373 SEOp = Mips::SEB;
374 [[fallthrough]];
375 case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
376 Opcode = Mips::AND;
377 break;
378 case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
379 SEOp = Mips::SEB;
380 [[fallthrough]];
381 case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
382 Opcode = Mips::OR;
383 break;
384 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
385 SEOp = Mips::SEB;
386 [[fallthrough]];
387 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
388 Opcode = Mips::XOR;
389 break;
390 case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
391 case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
392 IsUnsigned = true;
393 [[fallthrough]];
394 case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
395 case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
396 IsMin = true;
397 break;
398 case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
399 case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
400 IsUnsigned = true;
401 [[fallthrough]];
402 case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
403 case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
404 IsMax = true;
405 break;
406 default:
407 llvm_unreachable("Unknown subword atomic pseudo for expansion!");
408 }
409
410 Register Dest = I->getOperand(0).getReg();
411 Register Ptr = I->getOperand(1).getReg();
412 Register Incr = I->getOperand(2).getReg();
413 Register Mask = I->getOperand(3).getReg();
414 Register Mask2 = I->getOperand(4).getReg();
415 Register ShiftAmnt = I->getOperand(5).getReg();
416 Register OldVal = I->getOperand(6).getReg();
417 Register BinOpRes = I->getOperand(7).getReg();
418 Register StoreVal = I->getOperand(8).getReg();
419
420 const BasicBlock *LLVM_BB = BB.getBasicBlock();
421 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
422 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
423 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
425 MF->insert(It, loopMBB);
426 MF->insert(It, sinkMBB);
427 MF->insert(It, exitMBB);
428
429 exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
431
433 loopMBB->addSuccessor(sinkMBB);
434 loopMBB->addSuccessor(loopMBB);
435 loopMBB->normalizeSuccProbs();
436
437 BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
438 if (IsNand) {
439 // and andres, oldval, incr2
440 // nor binopres, $0, andres
441 // and newval, binopres, mask
442 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
443 .addReg(OldVal)
444 .addReg(Incr);
445 BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
446 .addReg(Mips::ZERO)
447 .addReg(BinOpRes);
448 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
449 .addReg(BinOpRes)
450 .addReg(Mask);
451 } else if (IsMin || IsMax) {
452
453 assert(I->getNumOperands() == 10 &&
454 "Atomics min|max|umin|umax use an additional register");
455 Register Scratch4 = I->getOperand(9).getReg();
456
457 unsigned SLTScratch4 = IsUnsigned ? SLTu : SLT;
458 unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
459 unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
460 unsigned MOVIncr = IsMax ? MOVN : MOVZ;
461
462 // For little endian we need to clear uninterested bits.
463 if (STI->isLittle()) {
464 // and OldVal, OldVal, Mask
465 // and Incr, Incr, Mask
466 BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal)
467 .addReg(OldVal)
468 .addReg(Mask);
469 BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr).addReg(Incr).addReg(Mask);
470 }
471
472 // unsigned: sltu Scratch4, oldVal, Incr
473 // signed: slt Scratch4, oldVal, Incr
474 BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4)
475 .addReg(OldVal)
476 .addReg(Incr);
477
478 if (STI->hasMips64r6() || STI->hasMips32r6()) {
479 // max: seleqz BinOpRes, OldVal, Scratch4
480 // selnez Scratch4, Incr, Scratch4
481 // or BinOpRes, BinOpRes, Scratch4
482 // min: selnqz BinOpRes, OldVal, Scratch4
483 // seleqz Scratch4, Incr, Scratch4
484 // or BinOpRes, BinOpRes, Scratch4
485 BuildMI(loopMBB, DL, TII->get(SELOldVal), BinOpRes)
486 .addReg(OldVal)
487 .addReg(Scratch4);
488 BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch4)
489 .addReg(Incr)
490 .addReg(Scratch4);
491 BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
492 .addReg(BinOpRes)
493 .addReg(Scratch4);
494 } else {
495 // max: move BinOpRes, OldVal
496 // movn BinOpRes, Incr, Scratch4, BinOpRes
497 // min: move BinOpRes, OldVal
498 // movz BinOpRes, Incr, Scratch4, BinOpRes
499 BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
500 .addReg(OldVal)
501 .addReg(Mips::ZERO);
502 BuildMI(loopMBB, DL, TII->get(MOVIncr), BinOpRes)
503 .addReg(Incr)
504 .addReg(Scratch4)
505 .addReg(BinOpRes);
506 }
507
508 // and BinOpRes, BinOpRes, Mask
509 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
510 .addReg(BinOpRes)
511 .addReg(Mask);
512
513 } else if (!IsSwap) {
514 // <binop> binopres, oldval, incr2
515 // and newval, binopres, mask
516 BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
517 .addReg(OldVal)
518 .addReg(Incr);
519 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
520 .addReg(BinOpRes)
521 .addReg(Mask);
522 } else { // atomic.swap
523 // and newval, incr2, mask
524 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
525 .addReg(Incr)
526 .addReg(Mask);
527 }
528
529 // and StoreVal, OlddVal, Mask2
530 // or StoreVal, StoreVal, BinOpRes
531 // StoreVal<tied1> = sc StoreVal, 0(Ptr)
532 // beq StoreVal, zero, loopMBB
533 BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
534 .addReg(OldVal).addReg(Mask2);
535 BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
536 .addReg(StoreVal).addReg(BinOpRes);
537 BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
538 .addReg(StoreVal).addReg(Ptr).addImm(0);
539 BuildMI(loopMBB, DL, TII->get(BEQ))
540 .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
541
542 // sinkMBB:
543 // and maskedoldval1,oldval,mask
544 // srl srlres,maskedoldval1,shiftamt
545 // sign_extend dest,srlres
546
547 sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
548
549 BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest)
550 .addReg(OldVal).addReg(Mask);
551 BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
552 .addReg(Dest).addReg(ShiftAmnt);
553
554 if (STI->hasMips32r2()) {
555 BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
556 } else {
557 const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
558 BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
559 .addReg(Dest, RegState::Kill)
560 .addImm(ShiftImm);
561 BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
562 .addReg(Dest, RegState::Kill)
563 .addImm(ShiftImm);
564 }
565
566 LivePhysRegs LiveRegs;
567 computeAndAddLiveIns(LiveRegs, *loopMBB);
568 computeAndAddLiveIns(LiveRegs, *sinkMBB);
569 computeAndAddLiveIns(LiveRegs, *exitMBB);
570
571 NMBBI = BB.end();
572 I->eraseFromParent();
573
574 return true;
575}
576
577bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
580 unsigned Size) {
581 MachineFunction *MF = BB.getParent();
582
583 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
584 DebugLoc DL = I->getDebugLoc();
585
586 unsigned LL, SC, ZERO, BEQ, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
587
588 if (Size == 4) {
589 if (STI->inMicroMipsMode()) {
590 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
591 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
592 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
593 SLT = Mips::SLT_MM;
594 SLTu = Mips::SLTu_MM;
595 OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
596 MOVN = Mips::MOVN_I_MM;
597 MOVZ = Mips::MOVZ_I_MM;
598 SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
599 SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
600 } else {
601 LL = STI->hasMips32r6()
602 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
603 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
604 SC = STI->hasMips32r6()
605 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
606 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
607 BEQ = Mips::BEQ;
608 SLT = Mips::SLT;
609 SLTu = Mips::SLTu;
610 OR = Mips::OR;
611 MOVN = Mips::MOVN_I_I;
612 MOVZ = Mips::MOVZ_I_I;
613 SELNEZ = Mips::SELNEZ;
614 SELEQZ = Mips::SELEQZ;
615 }
616
617 ZERO = Mips::ZERO;
618 } else {
619 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
620 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
621 ZERO = Mips::ZERO_64;
622 BEQ = Mips::BEQ64;
623 SLT = Mips::SLT64;
624 SLTu = Mips::SLTu64;
625 OR = Mips::OR64;
626 MOVN = Mips::MOVN_I64_I64;
627 MOVZ = Mips::MOVZ_I64_I64;
628 SELNEZ = Mips::SELNEZ64;
629 SELEQZ = Mips::SELEQZ64;
630 }
631
632 Register OldVal = I->getOperand(0).getReg();
633 Register Ptr = I->getOperand(1).getReg();
634 Register Incr = I->getOperand(2).getReg();
635 Register Scratch = I->getOperand(3).getReg();
636
637 unsigned Opcode = 0;
638 unsigned AND = 0;
639 unsigned NOR = 0;
640
641 bool IsOr = false;
642 bool IsNand = false;
643 bool IsMin = false;
644 bool IsMax = false;
645 bool IsUnsigned = false;
646
647 switch (I->getOpcode()) {
648 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
649 Opcode = Mips::ADDu;
650 break;
651 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
652 Opcode = Mips::SUBu;
653 break;
654 case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
655 Opcode = Mips::AND;
656 break;
657 case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
658 Opcode = Mips::OR;
659 break;
660 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
661 Opcode = Mips::XOR;
662 break;
663 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
664 IsNand = true;
665 AND = Mips::AND;
666 NOR = Mips::NOR;
667 break;
668 case Mips::ATOMIC_SWAP_I32_POSTRA:
669 IsOr = true;
670 break;
671 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
672 Opcode = Mips::DADDu;
673 break;
674 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
675 Opcode = Mips::DSUBu;
676 break;
677 case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
678 Opcode = Mips::AND64;
679 break;
680 case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
681 Opcode = Mips::OR64;
682 break;
683 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
684 Opcode = Mips::XOR64;
685 break;
686 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
687 IsNand = true;
688 AND = Mips::AND64;
689 NOR = Mips::NOR64;
690 break;
691 case Mips::ATOMIC_SWAP_I64_POSTRA:
692 IsOr = true;
693 break;
694 case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
695 case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
696 IsUnsigned = true;
697 [[fallthrough]];
698 case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
699 case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
700 IsMin = true;
701 break;
702 case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
703 case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
704 IsUnsigned = true;
705 [[fallthrough]];
706 case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
707 case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
708 IsMax = true;
709 break;
710 default:
711 llvm_unreachable("Unknown pseudo atomic!");
712 }
713
714 const BasicBlock *LLVM_BB = BB.getBasicBlock();
715 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
716 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
718 MF->insert(It, loopMBB);
719 MF->insert(It, exitMBB);
720
721 exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
723
725 loopMBB->addSuccessor(exitMBB);
726 loopMBB->addSuccessor(loopMBB);
727 loopMBB->normalizeSuccProbs();
728
729 BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
730 assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
731 assert((OldVal != Incr) && "Clobbered the wrong reg!");
732 if (IsMin || IsMax) {
733
734 assert(I->getNumOperands() == 5 &&
735 "Atomics min|max|umin|umax use an additional register");
736 MCRegister Scratch2 = I->getOperand(4).getReg().asMCReg();
737
738 // On Mips64 result of slt is GPR32.
739 MCRegister Scratch2_32 =
740 (Size == 8) ? STI->getRegisterInfo()->getSubReg(Scratch2, Mips::sub_32)
741 : Scratch2;
742
743 unsigned SLTScratch2 = IsUnsigned ? SLTu : SLT;
744 unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
745 unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
746 unsigned MOVIncr = IsMax ? MOVN : MOVZ;
747
748 // unsigned: sltu Scratch2, oldVal, Incr
749 // signed: slt Scratch2, oldVal, Incr
750 BuildMI(loopMBB, DL, TII->get(SLTScratch2), Scratch2_32)
751 .addReg(OldVal)
752 .addReg(Incr);
753
754 if (STI->hasMips64r6() || STI->hasMips32r6()) {
755 // max: seleqz Scratch, OldVal, Scratch2
756 // selnez Scratch2, Incr, Scratch2
757 // or Scratch, Scratch, Scratch2
758 // min: selnez Scratch, OldVal, Scratch2
759 // seleqz Scratch2, Incr, Scratch2
760 // or Scratch, Scratch, Scratch2
761 BuildMI(loopMBB, DL, TII->get(SELOldVal), Scratch)
762 .addReg(OldVal)
763 .addReg(Scratch2);
764 BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch2)
765 .addReg(Incr)
766 .addReg(Scratch2);
767 BuildMI(loopMBB, DL, TII->get(OR), Scratch)
768 .addReg(Scratch)
769 .addReg(Scratch2);
770 } else {
771 // max: move Scratch, OldVal
772 // movn Scratch, Incr, Scratch2, Scratch
773 // min: move Scratch, OldVal
774 // movz Scratch, Incr, Scratch2, Scratch
775 BuildMI(loopMBB, DL, TII->get(OR), Scratch)
776 .addReg(OldVal)
777 .addReg(ZERO);
778 BuildMI(loopMBB, DL, TII->get(MOVIncr), Scratch)
779 .addReg(Incr)
780 .addReg(Scratch2)
781 .addReg(Scratch);
782 }
783
784 } else if (Opcode) {
785 BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
786 } else if (IsNand) {
787 assert(AND && NOR &&
788 "Unknown nand instruction for atomic pseudo expansion");
789 BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
790 BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
791 } else {
792 assert(IsOr && OR && "Unknown instruction for atomic pseudo expansion!");
793 (void)IsOr;
794 BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
795 }
796
797 BuildMI(loopMBB, DL, TII->get(SC), Scratch)
798 .addReg(Scratch)
799 .addReg(Ptr)
800 .addImm(0);
801 BuildMI(loopMBB, DL, TII->get(BEQ))
802 .addReg(Scratch)
803 .addReg(ZERO)
804 .addMBB(loopMBB);
805
806 NMBBI = BB.end();
807 I->eraseFromParent();
808
809 LivePhysRegs LiveRegs;
810 computeAndAddLiveIns(LiveRegs, *loopMBB);
811 computeAndAddLiveIns(LiveRegs, *exitMBB);
812
813 return true;
814}
815
816bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB,
819
820 bool Modified = false;
821
822 switch (MBBI->getOpcode()) {
823 case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
824 case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
825 return expandAtomicCmpSwap(MBB, MBBI, NMBB);
826 case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
827 case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
828 return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB);
829 case Mips::ATOMIC_SWAP_I8_POSTRA:
830 case Mips::ATOMIC_SWAP_I16_POSTRA:
831 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
832 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
833 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
834 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
835 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
836 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
837 case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
838 case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
839 case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
840 case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
841 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
842 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
843 case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
844 case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
845 case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
846 case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
847 case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
848 case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
849 case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
850 case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
851 return expandAtomicBinOpSubword(MBB, MBBI, NMBB);
852 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
853 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
854 case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
855 case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
856 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
857 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
858 case Mips::ATOMIC_SWAP_I32_POSTRA:
859 case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
860 case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
861 case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
862 case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
863 return expandAtomicBinOp(MBB, MBBI, NMBB, 4);
864 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
865 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
866 case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
867 case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
868 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
869 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
870 case Mips::ATOMIC_SWAP_I64_POSTRA:
871 case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
872 case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
873 case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
874 case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
875 return expandAtomicBinOp(MBB, MBBI, NMBB, 8);
876 default:
877 return Modified;
878 }
879}
880
881bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
882 bool Modified = false;
883
885 while (MBBI != E) {
886 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
887 Modified |= expandMI(MBB, MBBI, NMBBI);
888 MBBI = NMBBI;
889 }
890
891 return Modified;
892}
893
894bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
895 STI = &MF.getSubtarget<MipsSubtarget>();
896 TII = STI->getInstrInfo();
897
898 bool Modified = false;
899 for (MachineBasicBlock &MBB : MF)
900 Modified |= expandMBB(MBB);
901
902 if (Modified)
903 MF.RenumberBlocks();
904
905 return Modified;
906}
907
908/// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
909/// expansion pass.
911 return new MipsExpandPseudo();
912}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
@ ZERO
Special weight used for cases with exact zero probability.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
uint64_t Size
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
static BranchProbability getOne()
A debug info location.
Definition: DebugLoc.h:33
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:50
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:24
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
self_iterator getIterator()
Definition: ilist_node.h:82
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
@ Kill
The last use of a register.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
FunctionPass * createMipsExpandPseudoPass()
createMipsExpandPseudoPass - returns an instance of the pseudo instruction expansion pass.