LLVM  16.0.0git
InstCombineAtomicRMW.cpp
Go to the documentation of this file.
1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the visit functions for atomic rmw instructions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "InstCombineInternal.h"
14 #include "llvm/IR/Instructions.h"
15 
16 using namespace llvm;
17 
18 namespace {
19 /// Return true if and only if the given instruction does not modify the memory
20 /// location referenced. Note that an idemptent atomicrmw may still have
21 /// ordering effects on nearby instructions, or be volatile.
22 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
23 /// Idemptotent is confusing in this context.
24 bool isIdempotentRMW(AtomicRMWInst& RMWI) {
25  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
26  switch(RMWI.getOperation()) {
27  case AtomicRMWInst::FAdd: // -0.0
28  return CF->isZero() && CF->isNegative();
29  case AtomicRMWInst::FSub: // +0.0
30  return CF->isZero() && !CF->isNegative();
31  default:
32  return false;
33  };
34 
35  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
36  if(!C)
37  return false;
38 
39  switch(RMWI.getOperation()) {
40  case AtomicRMWInst::Add:
41  case AtomicRMWInst::Sub:
42  case AtomicRMWInst::Or:
43  case AtomicRMWInst::Xor:
44  return C->isZero();
45  case AtomicRMWInst::And:
46  return C->isMinusOne();
47  case AtomicRMWInst::Min:
48  return C->isMaxValue(true);
49  case AtomicRMWInst::Max:
50  return C->isMinValue(true);
52  return C->isMaxValue(false);
54  return C->isMinValue(false);
55  default:
56  return false;
57  }
58 }
59 
60 /// Return true if the given instruction always produces a value in memory
61 /// equivalent to its value operand.
62 bool isSaturating(AtomicRMWInst& RMWI) {
63  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
64  switch (RMWI.getOperation()) {
66  // maxnum(x, +inf) -> +inf
67  return !CF->isNegative() && CF->isInfinity();
69  // minnum(x, -inf) -> +inf
70  return CF->isNegative() && CF->isInfinity();
73  return CF->isNaN();
74  default:
75  return false;
76  };
77 
78  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
79  if(!C)
80  return false;
81 
82  switch(RMWI.getOperation()) {
83  default:
84  return false;
86  return true;
87  case AtomicRMWInst::Or:
88  return C->isAllOnesValue();
89  case AtomicRMWInst::And:
90  return C->isZero();
91  case AtomicRMWInst::Min:
92  return C->isMinValue(true);
93  case AtomicRMWInst::Max:
94  return C->isMaxValue(true);
96  return C->isMinValue(false);
98  return C->isMaxValue(false);
99  };
100 }
101 } // namespace
102 
104 
105  // Volatile RMWs perform a load and a store, we cannot replace this by just a
106  // load or just a store. We chose not to canonicalize out of general paranoia
107  // about user expectations around volatile.
108  if (RMWI.isVolatile())
109  return nullptr;
110 
111  // Any atomicrmw op which produces a known result in memory can be
112  // replaced w/an atomicrmw xchg.
113  if (isSaturating(RMWI) &&
114  RMWI.getOperation() != AtomicRMWInst::Xchg) {
116  return &RMWI;
117  }
118 
119  AtomicOrdering Ordering = RMWI.getOrdering();
120  assert(Ordering != AtomicOrdering::NotAtomic &&
121  Ordering != AtomicOrdering::Unordered &&
122  "AtomicRMWs don't make sense with Unordered or NotAtomic");
123 
124  // Any atomicrmw xchg with no uses can be converted to a atomic store if the
125  // ordering is compatible.
126  if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
127  RMWI.use_empty()) {
128  if (Ordering != AtomicOrdering::Release &&
129  Ordering != AtomicOrdering::Monotonic)
130  return nullptr;
131  auto *SI = new StoreInst(RMWI.getValOperand(),
132  RMWI.getPointerOperand(), &RMWI);
133  SI->setAtomic(Ordering, RMWI.getSyncScopeID());
134  SI->setAlignment(DL.getABITypeAlign(RMWI.getType()));
135  return eraseInstFromFunction(RMWI);
136  }
137 
138  if (!isIdempotentRMW(RMWI))
139  return nullptr;
140 
141  // We chose to canonicalize all idempotent operations to an single
142  // operation code and constant. This makes it easier for the rest of the
143  // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are
144  // arbitrary.
145  if (RMWI.getType()->isIntegerTy() &&
146  RMWI.getOperation() != AtomicRMWInst::Or) {
148  return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));
149  } else if (RMWI.getType()->isFloatingPointTy() &&
150  RMWI.getOperation() != AtomicRMWInst::FAdd) {
152  return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));
153  }
154 
155  // Check if the required ordering is compatible with an atomic load.
156  if (Ordering != AtomicOrdering::Acquire &&
157  Ordering != AtomicOrdering::Monotonic)
158  return nullptr;
159 
160  LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand(), "",
161  false, DL.getABITypeAlign(RMWI.getType()),
162  Ordering, RMWI.getSyncScopeID());
163  return Load;
164 }
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::AtomicRMWInst::setOperation
void setOperation(BinOp Operation)
Definition: Instructions.h:816
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:740
llvm::AtomicRMWInst::getOperation
BinOp getOperation() const
Definition: Instructions.h:800
llvm::ConstantFP::getNegativeZero
static Constant * getNegativeZero(Type *Ty)
Definition: Constants.h:293
llvm::Type::isFloatingPointTy
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition: Type.h:184
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:762
llvm::AtomicOrdering::Monotonic
@ Monotonic
llvm::InstCombinerImpl::visitAtomicRMWInst
Instruction * visitAtomicRMWInst(AtomicRMWInst &SI)
Definition: InstCombineAtomicRMW.cpp:103
llvm::AtomicRMWInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:865
llvm::InstCombinerImpl::eraseInstFromFunction
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Definition: InstCombineInternal.h:456
InstCombineInternal.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
SI
@ SI
Definition: SIInstrInfo.cpp:7882
llvm::Instruction
Definition: Instruction.h:42
llvm::DataLayout::getABITypeAlign
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:829
llvm::AtomicOrdering::Acquire
@ Acquire
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:758
llvm::AtomicRMWInst::Xchg
@ Xchg
*p = v
Definition: Instructions.h:728
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:730
llvm::Value::use_empty
bool use_empty() const
Definition: Value.h:344
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:210
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition: AtomicOrdering.h:56
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:748
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:297
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:732
llvm::AtomicRMWInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:842
llvm::AtomicOrdering::Unordered
@ Unordered
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:744
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:738
llvm::AtomicRMWInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
Definition: Instructions.h:856
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:714
llvm::AtomicOrdering::Release
@ Release
llvm::AtomicRMWInst::isVolatile
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
Definition: Instructions.h:832
llvm::AtomicRMWInst::FSub
@ FSub
*p = old - v
Definition: Instructions.h:754
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::InstCombiner::DL
const DataLayout & DL
Definition: InstCombiner.h:73
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:734
llvm::AtomicRMWInst::getValOperand
Value * getValOperand()
Definition: Instructions.h:869
llvm::InstCombinerImpl::replaceOperand
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombineInternal.h:429
Instructions.h
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition: Instructions.h:751
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:746
llvm::AtomicOrdering::NotAtomic
@ NotAtomic
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:742