clang  3.9.0
CGAtomic.cpp
Go to the documentation of this file.
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CGCall.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Operator.h"
24 
25 using namespace clang;
26 using namespace CodeGen;
27 
28 namespace {
29  class AtomicInfo {
30  CodeGenFunction &CGF;
31  QualType AtomicTy;
32  QualType ValueTy;
33  uint64_t AtomicSizeInBits;
34  uint64_t ValueSizeInBits;
35  CharUnits AtomicAlign;
36  CharUnits ValueAlign;
37  CharUnits LValueAlign;
38  TypeEvaluationKind EvaluationKind;
39  bool UseLibcall;
40  LValue LVal;
41  CGBitFieldInfo BFI;
42  public:
43  AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44  : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45  EvaluationKind(TEK_Scalar), UseLibcall(true) {
46  assert(!lvalue.isGlobalReg());
47  ASTContext &C = CGF.getContext();
48  if (lvalue.isSimple()) {
49  AtomicTy = lvalue.getType();
50  if (auto *ATy = AtomicTy->getAs<AtomicType>())
51  ValueTy = ATy->getValueType();
52  else
53  ValueTy = AtomicTy;
54  EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56  uint64_t ValueAlignInBits;
57  uint64_t AtomicAlignInBits;
58  TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59  ValueSizeInBits = ValueTI.Width;
60  ValueAlignInBits = ValueTI.Align;
61 
62  TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63  AtomicSizeInBits = AtomicTI.Width;
64  AtomicAlignInBits = AtomicTI.Align;
65 
66  assert(ValueSizeInBits <= AtomicSizeInBits);
67  assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69  AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70  ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71  if (lvalue.getAlignment().isZero())
72  lvalue.setAlignment(AtomicAlign);
73 
74  LVal = lvalue;
75  } else if (lvalue.isBitField()) {
76  ValueTy = lvalue.getType();
77  ValueSizeInBits = C.getTypeSize(ValueTy);
78  auto &OrigBFI = lvalue.getBitFieldInfo();
79  auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80  AtomicSizeInBits = C.toBits(
81  C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82  .alignTo(lvalue.getAlignment()));
83  auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84  auto OffsetInChars =
85  (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86  lvalue.getAlignment();
87  VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88  VoidPtrAddr, OffsetInChars.getQuantity());
90  VoidPtrAddr,
91  CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92  "atomic_bitfield_base");
93  BFI = OrigBFI;
94  BFI.Offset = Offset;
95  BFI.StorageSize = AtomicSizeInBits;
96  BFI.StorageOffset += OffsetInChars;
97  LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98  BFI, lvalue.getType(),
99  lvalue.getAlignmentSource());
100  LVal.setTBAAInfo(lvalue.getTBAAInfo());
101  AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102  if (AtomicTy.isNull()) {
103  llvm::APInt Size(
104  /*numBits=*/32,
105  C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106  AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107  /*IndexTypeQuals=*/0);
108  }
109  AtomicAlign = ValueAlign = lvalue.getAlignment();
110  } else if (lvalue.isVectorElt()) {
111  ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112  ValueSizeInBits = C.getTypeSize(ValueTy);
113  AtomicTy = lvalue.getType();
114  AtomicSizeInBits = C.getTypeSize(AtomicTy);
115  AtomicAlign = ValueAlign = lvalue.getAlignment();
116  LVal = lvalue;
117  } else {
118  assert(lvalue.isExtVectorElt());
119  ValueTy = lvalue.getType();
120  ValueSizeInBits = C.getTypeSize(ValueTy);
121  AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122  lvalue.getType(), lvalue.getExtVectorAddress()
123  .getElementType()->getVectorNumElements());
124  AtomicSizeInBits = C.getTypeSize(AtomicTy);
125  AtomicAlign = ValueAlign = lvalue.getAlignment();
126  LVal = lvalue;
127  }
128  UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129  AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130  }
131 
132  QualType getAtomicType() const { return AtomicTy; }
133  QualType getValueType() const { return ValueTy; }
134  CharUnits getAtomicAlignment() const { return AtomicAlign; }
135  CharUnits getValueAlignment() const { return ValueAlign; }
136  uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137  uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138  TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139  bool shouldUseLibcall() const { return UseLibcall; }
140  const LValue &getAtomicLValue() const { return LVal; }
141  llvm::Value *getAtomicPointer() const {
142  if (LVal.isSimple())
143  return LVal.getPointer();
144  else if (LVal.isBitField())
145  return LVal.getBitFieldPointer();
146  else if (LVal.isVectorElt())
147  return LVal.getVectorPointer();
148  assert(LVal.isExtVectorElt());
149  return LVal.getExtVectorPointer();
150  }
151  Address getAtomicAddress() const {
152  return Address(getAtomicPointer(), getAtomicAlignment());
153  }
154 
155  Address getAtomicAddressAsAtomicIntPointer() const {
156  return emitCastToAtomicIntPointer(getAtomicAddress());
157  }
158 
159  /// Is the atomic size larger than the underlying value type?
160  ///
161  /// Note that the absence of padding does not mean that atomic
162  /// objects are completely interchangeable with non-atomic
163  /// objects: we might have promoted the alignment of a type
164  /// without making it bigger.
165  bool hasPadding() const {
166  return (ValueSizeInBits != AtomicSizeInBits);
167  }
168 
169  bool emitMemSetZeroIfNecessary() const;
170 
171  llvm::Value *getAtomicSizeValue() const {
172  CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173  return CGF.CGM.getSize(size);
174  }
175 
176  /// Cast the given pointer to an integer pointer suitable for atomic
177  /// operations if the source.
178  Address emitCastToAtomicIntPointer(Address Addr) const;
179 
180  /// If Addr is compatible with the iN that will be used for an atomic
181  /// operation, bitcast it. Otherwise, create a temporary that is suitable
182  /// and copy the value across.
183  Address convertToAtomicIntPointer(Address Addr) const;
184 
185  /// Turn an atomic-layout object into an r-value.
186  RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187  SourceLocation loc, bool AsValue) const;
188 
189  /// \brief Converts a rvalue to integer value.
190  llvm::Value *convertRValueToInt(RValue RVal) const;
191 
192  RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193  AggValueSlot ResultSlot,
194  SourceLocation Loc, bool AsValue) const;
195 
196  /// Copy an atomic r-value into atomic-layout memory.
197  void emitCopyIntoMemory(RValue rvalue) const;
198 
199  /// Project an l-value down to the value field.
200  LValue projectValue() const {
201  assert(LVal.isSimple());
202  Address addr = getAtomicAddress();
203  if (hasPadding())
204  addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
205 
206  return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207  LVal.getAlignmentSource(), LVal.getTBAAInfo());
208  }
209 
210  /// \brief Emits atomic load.
211  /// \returns Loaded value.
212  RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213  bool AsValue, llvm::AtomicOrdering AO,
214  bool IsVolatile);
215 
216  /// \brief Emits atomic compare-and-exchange sequence.
217  /// \param Expected Expected value.
218  /// \param Desired Desired value.
219  /// \param Success Atomic ordering for success operation.
220  /// \param Failure Atomic ordering for failed operation.
221  /// \param IsWeak true if atomic operation is weak, false otherwise.
222  /// \returns Pair of values: previous value from storage (value type) and
223  /// boolean flag (i1 type) with true if success and false otherwise.
224  std::pair<RValue, llvm::Value *>
225  EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226  llvm::AtomicOrdering Success =
227  llvm::AtomicOrdering::SequentiallyConsistent,
228  llvm::AtomicOrdering Failure =
229  llvm::AtomicOrdering::SequentiallyConsistent,
230  bool IsWeak = false);
231 
232  /// \brief Emits atomic update.
233  /// \param AO Atomic ordering.
234  /// \param UpdateOp Update operation for the current lvalue.
235  void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236  const llvm::function_ref<RValue(RValue)> &UpdateOp,
237  bool IsVolatile);
238  /// \brief Emits atomic update.
239  /// \param AO Atomic ordering.
240  void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241  bool IsVolatile);
242 
243  /// Materialize an atomic r-value in atomic-layout memory.
244  Address materializeRValue(RValue rvalue) const;
245 
246  /// \brief Creates temp alloca for intermediate operations on atomic value.
247  Address CreateTempAlloca() const;
248  private:
249  bool requiresMemSetZero(llvm::Type *type) const;
250 
251 
252  /// \brief Emits atomic load as a libcall.
253  void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254  llvm::AtomicOrdering AO, bool IsVolatile);
255  /// \brief Emits atomic load as LLVM instruction.
256  llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257  /// \brief Emits atomic compare-and-exchange op as a libcall.
258  llvm::Value *EmitAtomicCompareExchangeLibcall(
259  llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260  llvm::AtomicOrdering Success =
261  llvm::AtomicOrdering::SequentiallyConsistent,
262  llvm::AtomicOrdering Failure =
263  llvm::AtomicOrdering::SequentiallyConsistent);
264  /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
265  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267  llvm::AtomicOrdering Success =
268  llvm::AtomicOrdering::SequentiallyConsistent,
269  llvm::AtomicOrdering Failure =
270  llvm::AtomicOrdering::SequentiallyConsistent,
271  bool IsWeak = false);
272  /// \brief Emit atomic update as libcalls.
273  void
274  EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275  const llvm::function_ref<RValue(RValue)> &UpdateOp,
276  bool IsVolatile);
277  /// \brief Emit atomic update as LLVM instructions.
278  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279  const llvm::function_ref<RValue(RValue)> &UpdateOp,
280  bool IsVolatile);
281  /// \brief Emit atomic update as libcalls.
282  void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283  bool IsVolatile);
284  /// \brief Emit atomic update as LLVM instructions.
285  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286  bool IsVolatile);
287  };
288 }
289 
290 Address AtomicInfo::CreateTempAlloca() const {
291  Address TempAlloca = CGF.CreateMemTemp(
292  (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293  : AtomicTy,
294  getAtomicAlignment(),
295  "atomic-temp");
296  // Cast to pointer to value type for bitfields.
297  if (LVal.isBitField())
299  TempAlloca, getAtomicAddress().getType());
300  return TempAlloca;
301 }
302 
304  StringRef fnName,
305  QualType resultType,
306  CallArgList &args) {
307  const CGFunctionInfo &fnInfo =
308  CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310  llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311  return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
312 }
313 
314 /// Does a store of the given IR type modify the full expected width?
316  uint64_t expectedSize) {
317  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
318 }
319 
320 /// Does the atomic type require memsetting to zero before initialization?
321 ///
322 /// The IR type is provided as a way of making certain queries faster.
323 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
324  // If the atomic type has size padding, we definitely need a memset.
325  if (hasPadding()) return true;
326 
327  // Otherwise, do some simple heuristics to try to avoid it:
328  switch (getEvaluationKind()) {
329  // For scalars and complexes, check whether the store size of the
330  // type uses the full size.
331  case TEK_Scalar:
332  return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
333  case TEK_Complex:
334  return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
335  AtomicSizeInBits / 2);
336 
337  // Padding in structs has an undefined bit pattern. User beware.
338  case TEK_Aggregate:
339  return false;
340  }
341  llvm_unreachable("bad evaluation kind");
342 }
343 
344 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
345  assert(LVal.isSimple());
346  llvm::Value *addr = LVal.getPointer();
347  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
348  return false;
349 
350  CGF.Builder.CreateMemSet(
351  addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
352  CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
353  LVal.getAlignment().getQuantity());
354  return true;
355 }
356 
357 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
358  Address Dest, Address Ptr,
359  Address Val1, Address Val2,
360  uint64_t Size,
361  llvm::AtomicOrdering SuccessOrder,
362  llvm::AtomicOrdering FailureOrder) {
363  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
364  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
366 
367  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
368  Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
369  Pair->setVolatile(E->isVolatile());
370  Pair->setWeak(IsWeak);
371 
372  // Cmp holds the result of the compare-exchange operation: true on success,
373  // false on failure.
374  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
375  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
376 
377  // This basic block is used to hold the store instruction if the operation
378  // failed.
379  llvm::BasicBlock *StoreExpectedBB =
380  CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
381 
382  // This basic block is the exit point of the operation, we should end up
383  // here regardless of whether or not the operation succeeded.
384  llvm::BasicBlock *ContinueBB =
385  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
386 
387  // Update Expected if Expected isn't equal to Old, otherwise branch to the
388  // exit point.
389  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
390 
391  CGF.Builder.SetInsertPoint(StoreExpectedBB);
392  // Update the memory at Expected with Old's value.
393  CGF.Builder.CreateStore(Old, Val1);
394  // Finally, branch to the exit point.
395  CGF.Builder.CreateBr(ContinueBB);
396 
397  CGF.Builder.SetInsertPoint(ContinueBB);
398  // Update the memory at Dest with Cmp's value.
399  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
400 }
401 
402 /// Given an ordering required on success, emit all possible cmpxchg
403 /// instructions to cope with the provided (but possibly only dynamically known)
404 /// FailureOrder.
406  bool IsWeak, Address Dest, Address Ptr,
407  Address Val1, Address Val2,
408  llvm::Value *FailureOrderVal,
409  uint64_t Size,
410  llvm::AtomicOrdering SuccessOrder) {
411  llvm::AtomicOrdering FailureOrder;
412  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
413  auto FOS = FO->getSExtValue();
414  if (!llvm::isValidAtomicOrderingCABI(FOS))
415  FailureOrder = llvm::AtomicOrdering::Monotonic;
416  else
417  switch ((llvm::AtomicOrderingCABI)FOS) {
418  case llvm::AtomicOrderingCABI::relaxed:
419  case llvm::AtomicOrderingCABI::release:
420  case llvm::AtomicOrderingCABI::acq_rel:
421  FailureOrder = llvm::AtomicOrdering::Monotonic;
422  break;
423  case llvm::AtomicOrderingCABI::consume:
424  case llvm::AtomicOrderingCABI::acquire:
425  FailureOrder = llvm::AtomicOrdering::Acquire;
426  break;
427  case llvm::AtomicOrderingCABI::seq_cst:
428  FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
429  break;
430  }
431  if (isStrongerThan(FailureOrder, SuccessOrder)) {
432  // Don't assert on undefined behavior "failure argument shall be no
433  // stronger than the success argument".
434  FailureOrder =
435  llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
436  }
437  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
438  FailureOrder);
439  return;
440  }
441 
442  // Create all the relevant BB's
443  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
444  *SeqCstBB = nullptr;
445  MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
446  if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
447  SuccessOrder != llvm::AtomicOrdering::Release)
448  AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
449  if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
450  SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
451 
452  llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
453 
454  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
455 
456  // Emit all the different atomics
457 
458  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
459  // doesn't matter unless someone is crazy enough to use something that
460  // doesn't fold to a constant for the ordering.
461  CGF.Builder.SetInsertPoint(MonotonicBB);
462  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
463  Size, SuccessOrder, llvm::AtomicOrdering::Monotonic);
464  CGF.Builder.CreateBr(ContBB);
465 
466  if (AcquireBB) {
467  CGF.Builder.SetInsertPoint(AcquireBB);
468  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
469  Size, SuccessOrder, llvm::AtomicOrdering::Acquire);
470  CGF.Builder.CreateBr(ContBB);
471  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
472  AcquireBB);
473  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
474  AcquireBB);
475  }
476  if (SeqCstBB) {
477  CGF.Builder.SetInsertPoint(SeqCstBB);
478  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
479  llvm::AtomicOrdering::SequentiallyConsistent);
480  CGF.Builder.CreateBr(ContBB);
481  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
482  SeqCstBB);
483  }
484 
485  CGF.Builder.SetInsertPoint(ContBB);
486 }
487 
489  Address Ptr, Address Val1, Address Val2,
490  llvm::Value *IsWeak, llvm::Value *FailureOrder,
491  uint64_t Size, llvm::AtomicOrdering Order) {
492  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
493  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
494 
495  switch (E->getOp()) {
496  case AtomicExpr::AO__c11_atomic_init:
497  llvm_unreachable("Already handled!");
498 
499  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
500  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
501  FailureOrder, Size, Order);
502  return;
503  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
504  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
505  FailureOrder, Size, Order);
506  return;
507  case AtomicExpr::AO__atomic_compare_exchange:
508  case AtomicExpr::AO__atomic_compare_exchange_n: {
509  if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
510  emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
511  Val1, Val2, FailureOrder, Size, Order);
512  } else {
513  // Create all the relevant BB's
514  llvm::BasicBlock *StrongBB =
515  CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
516  llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
517  llvm::BasicBlock *ContBB =
518  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
519 
520  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
521  SI->addCase(CGF.Builder.getInt1(false), StrongBB);
522 
523  CGF.Builder.SetInsertPoint(StrongBB);
524  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
525  FailureOrder, Size, Order);
526  CGF.Builder.CreateBr(ContBB);
527 
528  CGF.Builder.SetInsertPoint(WeakBB);
529  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
530  FailureOrder, Size, Order);
531  CGF.Builder.CreateBr(ContBB);
532 
533  CGF.Builder.SetInsertPoint(ContBB);
534  }
535  return;
536  }
537  case AtomicExpr::AO__c11_atomic_load:
538  case AtomicExpr::AO__atomic_load_n:
539  case AtomicExpr::AO__atomic_load: {
540  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
541  Load->setAtomic(Order);
542  Load->setVolatile(E->isVolatile());
543  CGF.Builder.CreateStore(Load, Dest);
544  return;
545  }
546 
547  case AtomicExpr::AO__c11_atomic_store:
548  case AtomicExpr::AO__atomic_store:
549  case AtomicExpr::AO__atomic_store_n: {
550  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
551  llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
552  Store->setAtomic(Order);
553  Store->setVolatile(E->isVolatile());
554  return;
555  }
556 
557  case AtomicExpr::AO__c11_atomic_exchange:
558  case AtomicExpr::AO__atomic_exchange_n:
559  case AtomicExpr::AO__atomic_exchange:
560  Op = llvm::AtomicRMWInst::Xchg;
561  break;
562 
563  case AtomicExpr::AO__atomic_add_fetch:
564  PostOp = llvm::Instruction::Add;
565  // Fall through.
566  case AtomicExpr::AO__c11_atomic_fetch_add:
567  case AtomicExpr::AO__atomic_fetch_add:
568  Op = llvm::AtomicRMWInst::Add;
569  break;
570 
571  case AtomicExpr::AO__atomic_sub_fetch:
572  PostOp = llvm::Instruction::Sub;
573  // Fall through.
574  case AtomicExpr::AO__c11_atomic_fetch_sub:
575  case AtomicExpr::AO__atomic_fetch_sub:
576  Op = llvm::AtomicRMWInst::Sub;
577  break;
578 
579  case AtomicExpr::AO__atomic_and_fetch:
580  PostOp = llvm::Instruction::And;
581  // Fall through.
582  case AtomicExpr::AO__c11_atomic_fetch_and:
583  case AtomicExpr::AO__atomic_fetch_and:
585  break;
586 
587  case AtomicExpr::AO__atomic_or_fetch:
588  PostOp = llvm::Instruction::Or;
589  // Fall through.
590  case AtomicExpr::AO__c11_atomic_fetch_or:
591  case AtomicExpr::AO__atomic_fetch_or:
592  Op = llvm::AtomicRMWInst::Or;
593  break;
594 
595  case AtomicExpr::AO__atomic_xor_fetch:
596  PostOp = llvm::Instruction::Xor;
597  // Fall through.
598  case AtomicExpr::AO__c11_atomic_fetch_xor:
599  case AtomicExpr::AO__atomic_fetch_xor:
600  Op = llvm::AtomicRMWInst::Xor;
601  break;
602 
603  case AtomicExpr::AO__atomic_nand_fetch:
604  PostOp = llvm::Instruction::And; // the NOT is special cased below
605  // Fall through.
606  case AtomicExpr::AO__atomic_fetch_nand:
607  Op = llvm::AtomicRMWInst::Nand;
608  break;
609  }
610 
611  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
612  llvm::AtomicRMWInst *RMWI =
613  CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
614  RMWI->setVolatile(E->isVolatile());
615 
616  // For __atomic_*_fetch operations, perform the operation again to
617  // determine the value which was written.
618  llvm::Value *Result = RMWI;
619  if (PostOp)
620  Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
621  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
622  Result = CGF.Builder.CreateNot(Result);
623  CGF.Builder.CreateStore(Result, Dest);
624 }
625 
626 // This function emits any expression (scalar, complex, or aggregate)
627 // into a temporary alloca.
628 static Address
630  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
631  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
632  /*Init*/ true);
633  return DeclPtr;
634 }
635 
636 static void
638  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
639  SourceLocation Loc, CharUnits SizeInChars) {
640  if (UseOptimizedLibcall) {
641  // Load value and pass it to the function directly.
642  CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
643  int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
644  ValTy =
645  CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
646  llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
647  SizeInBits)->getPointerTo();
648  Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
649  Val = CGF.EmitLoadOfScalar(Ptr, false,
650  CGF.getContext().getPointerType(ValTy),
651  Loc);
652  // Coerce the value into an appropriately sized integer type.
653  Args.add(RValue::get(Val), ValTy);
654  } else {
655  // Non-optimized functions always take a reference.
656  Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
657  CGF.getContext().VoidPtrTy);
658  }
659 }
660 
662  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
663  QualType MemTy = AtomicTy;
664  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
665  MemTy = AT->getValueType();
666  CharUnits sizeChars, alignChars;
667  std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
668  uint64_t Size = sizeChars.getQuantity();
669  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
670  bool UseLibcall = (sizeChars != alignChars ||
671  getContext().toBits(sizeChars) > MaxInlineWidthInBits);
672 
673  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
674 
675  Address Val1 = Address::invalid();
676  Address Val2 = Address::invalid();
677  Address Dest = Address::invalid();
678  Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
679 
680  if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
681  LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
682  EmitAtomicInit(E->getVal1(), lvalue);
683  return RValue::get(nullptr);
684  }
685 
686  llvm::Value *Order = EmitScalarExpr(E->getOrder());
687 
688  switch (E->getOp()) {
689  case AtomicExpr::AO__c11_atomic_init:
690  llvm_unreachable("Already handled above with EmitAtomicInit!");
691 
692  case AtomicExpr::AO__c11_atomic_load:
693  case AtomicExpr::AO__atomic_load_n:
694  break;
695 
696  case AtomicExpr::AO__atomic_load:
697  Dest = EmitPointerWithAlignment(E->getVal1());
698  break;
699 
700  case AtomicExpr::AO__atomic_store:
701  Val1 = EmitPointerWithAlignment(E->getVal1());
702  break;
703 
704  case AtomicExpr::AO__atomic_exchange:
705  Val1 = EmitPointerWithAlignment(E->getVal1());
706  Dest = EmitPointerWithAlignment(E->getVal2());
707  break;
708 
709  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
710  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
711  case AtomicExpr::AO__atomic_compare_exchange_n:
712  case AtomicExpr::AO__atomic_compare_exchange:
713  Val1 = EmitPointerWithAlignment(E->getVal1());
714  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
715  Val2 = EmitPointerWithAlignment(E->getVal2());
716  else
717  Val2 = EmitValToTemp(*this, E->getVal2());
718  OrderFail = EmitScalarExpr(E->getOrderFail());
719  if (E->getNumSubExprs() == 6)
720  IsWeak = EmitScalarExpr(E->getWeak());
721  break;
722 
723  case AtomicExpr::AO__c11_atomic_fetch_add:
724  case AtomicExpr::AO__c11_atomic_fetch_sub:
725  if (MemTy->isPointerType()) {
726  // For pointer arithmetic, we're required to do a bit of math:
727  // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
728  // ... but only for the C11 builtins. The GNU builtins expect the
729  // user to multiply by sizeof(T).
730  QualType Val1Ty = E->getVal1()->getType();
731  llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
732  CharUnits PointeeIncAmt =
734  Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
735  auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
736  Val1 = Temp;
737  EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
738  break;
739  }
740  // Fall through.
741  case AtomicExpr::AO__atomic_fetch_add:
742  case AtomicExpr::AO__atomic_fetch_sub:
743  case AtomicExpr::AO__atomic_add_fetch:
744  case AtomicExpr::AO__atomic_sub_fetch:
745  case AtomicExpr::AO__c11_atomic_store:
746  case AtomicExpr::AO__c11_atomic_exchange:
747  case AtomicExpr::AO__atomic_store_n:
748  case AtomicExpr::AO__atomic_exchange_n:
749  case AtomicExpr::AO__c11_atomic_fetch_and:
750  case AtomicExpr::AO__c11_atomic_fetch_or:
751  case AtomicExpr::AO__c11_atomic_fetch_xor:
752  case AtomicExpr::AO__atomic_fetch_and:
753  case AtomicExpr::AO__atomic_fetch_or:
754  case AtomicExpr::AO__atomic_fetch_xor:
755  case AtomicExpr::AO__atomic_fetch_nand:
756  case AtomicExpr::AO__atomic_and_fetch:
757  case AtomicExpr::AO__atomic_or_fetch:
758  case AtomicExpr::AO__atomic_xor_fetch:
759  case AtomicExpr::AO__atomic_nand_fetch:
760  Val1 = EmitValToTemp(*this, E->getVal1());
761  break;
762  }
763 
764  QualType RValTy = E->getType().getUnqualifiedType();
765 
766  // The inlined atomics only function on iN types, where N is a power of 2. We
767  // need to make sure (via temporaries if necessary) that all incoming values
768  // are compatible.
769  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
770  AtomicInfo Atomics(*this, AtomicVal);
771 
772  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
773  if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
774  if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
775  if (Dest.isValid())
776  Dest = Atomics.emitCastToAtomicIntPointer(Dest);
777  else if (E->isCmpXChg())
778  Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
779  else if (!RValTy->isVoidType())
780  Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
781 
782  // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
783  if (UseLibcall) {
784  bool UseOptimizedLibcall = false;
785  switch (E->getOp()) {
786  case AtomicExpr::AO__c11_atomic_init:
787  llvm_unreachable("Already handled above with EmitAtomicInit!");
788 
789  case AtomicExpr::AO__c11_atomic_fetch_add:
790  case AtomicExpr::AO__atomic_fetch_add:
791  case AtomicExpr::AO__c11_atomic_fetch_and:
792  case AtomicExpr::AO__atomic_fetch_and:
793  case AtomicExpr::AO__c11_atomic_fetch_or:
794  case AtomicExpr::AO__atomic_fetch_or:
795  case AtomicExpr::AO__atomic_fetch_nand:
796  case AtomicExpr::AO__c11_atomic_fetch_sub:
797  case AtomicExpr::AO__atomic_fetch_sub:
798  case AtomicExpr::AO__c11_atomic_fetch_xor:
799  case AtomicExpr::AO__atomic_fetch_xor:
800  case AtomicExpr::AO__atomic_add_fetch:
801  case AtomicExpr::AO__atomic_and_fetch:
802  case AtomicExpr::AO__atomic_nand_fetch:
803  case AtomicExpr::AO__atomic_or_fetch:
804  case AtomicExpr::AO__atomic_sub_fetch:
805  case AtomicExpr::AO__atomic_xor_fetch:
806  // For these, only library calls for certain sizes exist.
807  UseOptimizedLibcall = true;
808  break;
809 
810  case AtomicExpr::AO__c11_atomic_load:
811  case AtomicExpr::AO__c11_atomic_store:
812  case AtomicExpr::AO__c11_atomic_exchange:
813  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
814  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
815  case AtomicExpr::AO__atomic_load_n:
816  case AtomicExpr::AO__atomic_load:
817  case AtomicExpr::AO__atomic_store_n:
818  case AtomicExpr::AO__atomic_store:
819  case AtomicExpr::AO__atomic_exchange_n:
820  case AtomicExpr::AO__atomic_exchange:
821  case AtomicExpr::AO__atomic_compare_exchange_n:
822  case AtomicExpr::AO__atomic_compare_exchange:
823  // Only use optimized library calls for sizes for which they exist.
824  if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
825  UseOptimizedLibcall = true;
826  break;
827  }
828 
829  CallArgList Args;
830  if (!UseOptimizedLibcall) {
831  // For non-optimized library calls, the size is the first parameter
832  Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
833  getContext().getSizeType());
834  }
835  // Atomic address is the first or second parameter
838 
839  std::string LibCallName;
840  QualType LoweredMemTy =
841  MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
842  QualType RetTy;
843  bool HaveRetTy = false;
844  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
845  switch (E->getOp()) {
846  case AtomicExpr::AO__c11_atomic_init:
847  llvm_unreachable("Already handled!");
848 
849  // There is only one libcall for compare an exchange, because there is no
850  // optimisation benefit possible from a libcall version of a weak compare
851  // and exchange.
852  // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
853  // void *desired, int success, int failure)
854  // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
855  // int success, int failure)
856  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
857  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
858  case AtomicExpr::AO__atomic_compare_exchange:
859  case AtomicExpr::AO__atomic_compare_exchange_n:
860  LibCallName = "__atomic_compare_exchange";
861  RetTy = getContext().BoolTy;
862  HaveRetTy = true;
865  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
866  MemTy, E->getExprLoc(), sizeChars);
867  Args.add(RValue::get(Order), getContext().IntTy);
868  Order = OrderFail;
869  break;
870  // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
871  // int order)
872  // T __atomic_exchange_N(T *mem, T val, int order)
873  case AtomicExpr::AO__c11_atomic_exchange:
874  case AtomicExpr::AO__atomic_exchange_n:
875  case AtomicExpr::AO__atomic_exchange:
876  LibCallName = "__atomic_exchange";
877  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
878  MemTy, E->getExprLoc(), sizeChars);
879  break;
880  // void __atomic_store(size_t size, void *mem, void *val, int order)
881  // void __atomic_store_N(T *mem, T val, int order)
882  case AtomicExpr::AO__c11_atomic_store:
883  case AtomicExpr::AO__atomic_store:
884  case AtomicExpr::AO__atomic_store_n:
885  LibCallName = "__atomic_store";
886  RetTy = getContext().VoidTy;
887  HaveRetTy = true;
888  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
889  MemTy, E->getExprLoc(), sizeChars);
890  break;
891  // void __atomic_load(size_t size, void *mem, void *return, int order)
892  // T __atomic_load_N(T *mem, int order)
893  case AtomicExpr::AO__c11_atomic_load:
894  case AtomicExpr::AO__atomic_load:
895  case AtomicExpr::AO__atomic_load_n:
896  LibCallName = "__atomic_load";
897  break;
898  // T __atomic_add_fetch_N(T *mem, T val, int order)
899  // T __atomic_fetch_add_N(T *mem, T val, int order)
900  case AtomicExpr::AO__atomic_add_fetch:
901  PostOp = llvm::Instruction::Add;
902  // Fall through.
903  case AtomicExpr::AO__c11_atomic_fetch_add:
904  case AtomicExpr::AO__atomic_fetch_add:
905  LibCallName = "__atomic_fetch_add";
906  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
907  LoweredMemTy, E->getExprLoc(), sizeChars);
908  break;
909  // T __atomic_and_fetch_N(T *mem, T val, int order)
910  // T __atomic_fetch_and_N(T *mem, T val, int order)
911  case AtomicExpr::AO__atomic_and_fetch:
912  PostOp = llvm::Instruction::And;
913  // Fall through.
914  case AtomicExpr::AO__c11_atomic_fetch_and:
915  case AtomicExpr::AO__atomic_fetch_and:
916  LibCallName = "__atomic_fetch_and";
917  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
918  MemTy, E->getExprLoc(), sizeChars);
919  break;
920  // T __atomic_or_fetch_N(T *mem, T val, int order)
921  // T __atomic_fetch_or_N(T *mem, T val, int order)
922  case AtomicExpr::AO__atomic_or_fetch:
923  PostOp = llvm::Instruction::Or;
924  // Fall through.
925  case AtomicExpr::AO__c11_atomic_fetch_or:
926  case AtomicExpr::AO__atomic_fetch_or:
927  LibCallName = "__atomic_fetch_or";
928  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
929  MemTy, E->getExprLoc(), sizeChars);
930  break;
931  // T __atomic_sub_fetch_N(T *mem, T val, int order)
932  // T __atomic_fetch_sub_N(T *mem, T val, int order)
933  case AtomicExpr::AO__atomic_sub_fetch:
934  PostOp = llvm::Instruction::Sub;
935  // Fall through.
936  case AtomicExpr::AO__c11_atomic_fetch_sub:
937  case AtomicExpr::AO__atomic_fetch_sub:
938  LibCallName = "__atomic_fetch_sub";
939  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
940  LoweredMemTy, E->getExprLoc(), sizeChars);
941  break;
942  // T __atomic_xor_fetch_N(T *mem, T val, int order)
943  // T __atomic_fetch_xor_N(T *mem, T val, int order)
944  case AtomicExpr::AO__atomic_xor_fetch:
945  PostOp = llvm::Instruction::Xor;
946  // Fall through.
947  case AtomicExpr::AO__c11_atomic_fetch_xor:
948  case AtomicExpr::AO__atomic_fetch_xor:
949  LibCallName = "__atomic_fetch_xor";
950  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
951  MemTy, E->getExprLoc(), sizeChars);
952  break;
953  // T __atomic_nand_fetch_N(T *mem, T val, int order)
954  // T __atomic_fetch_nand_N(T *mem, T val, int order)
955  case AtomicExpr::AO__atomic_nand_fetch:
956  PostOp = llvm::Instruction::And; // the NOT is special cased below
957  // Fall through.
958  case AtomicExpr::AO__atomic_fetch_nand:
959  LibCallName = "__atomic_fetch_nand";
960  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
961  MemTy, E->getExprLoc(), sizeChars);
962  break;
963  }
964 
965  // Optimized functions have the size in their name.
966  if (UseOptimizedLibcall)
967  LibCallName += "_" + llvm::utostr(Size);
968  // By default, assume we return a value of the atomic type.
969  if (!HaveRetTy) {
970  if (UseOptimizedLibcall) {
971  // Value is returned directly.
972  // The function returns an appropriately sized integer type.
974  getContext().toBits(sizeChars), /*Signed=*/false);
975  } else {
976  // Value is returned through parameter before the order.
977  RetTy = getContext().VoidTy;
980  }
981  }
982  // order is always the last parameter
983  Args.add(RValue::get(Order),
984  getContext().IntTy);
985 
986  // PostOp is only needed for the atomic_*_fetch operations, and
987  // thus is only needed for and implemented in the
988  // UseOptimizedLibcall codepath.
989  assert(UseOptimizedLibcall || !PostOp);
990 
991  RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
992  // The value is returned directly from the libcall.
993  if (E->isCmpXChg())
994  return Res;
995 
996  // The value is returned directly for optimized libcalls but the expr
997  // provided an out-param.
998  if (UseOptimizedLibcall && Res.getScalarVal()) {
999  llvm::Value *ResVal = Res.getScalarVal();
1000  if (PostOp) {
1001  llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1002  ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1003  }
1004  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1005  ResVal = Builder.CreateNot(ResVal);
1006 
1008  ResVal,
1009  Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1010  }
1011 
1012  if (RValTy->isVoidType())
1013  return RValue::get(nullptr);
1014 
1015  return convertTempToRValue(
1016  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1017  RValTy, E->getExprLoc());
1018  }
1019 
1020  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1021  E->getOp() == AtomicExpr::AO__atomic_store ||
1022  E->getOp() == AtomicExpr::AO__atomic_store_n;
1023  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1024  E->getOp() == AtomicExpr::AO__atomic_load ||
1025  E->getOp() == AtomicExpr::AO__atomic_load_n;
1026 
1027  if (isa<llvm::ConstantInt>(Order)) {
1028  auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1029  // We should not ever get to a case where the ordering isn't a valid C ABI
1030  // value, but it's hard to enforce that in general.
1031  if (llvm::isValidAtomicOrderingCABI(ord))
1032  switch ((llvm::AtomicOrderingCABI)ord) {
1033  case llvm::AtomicOrderingCABI::relaxed:
1034  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1035  llvm::AtomicOrdering::Monotonic);
1036  break;
1037  case llvm::AtomicOrderingCABI::consume:
1038  case llvm::AtomicOrderingCABI::acquire:
1039  if (IsStore)
1040  break; // Avoid crashing on code with undefined behavior
1041  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1042  llvm::AtomicOrdering::Acquire);
1043  break;
1044  case llvm::AtomicOrderingCABI::release:
1045  if (IsLoad)
1046  break; // Avoid crashing on code with undefined behavior
1047  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1048  llvm::AtomicOrdering::Release);
1049  break;
1050  case llvm::AtomicOrderingCABI::acq_rel:
1051  if (IsLoad || IsStore)
1052  break; // Avoid crashing on code with undefined behavior
1053  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1054  llvm::AtomicOrdering::AcquireRelease);
1055  break;
1056  case llvm::AtomicOrderingCABI::seq_cst:
1057  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1058  llvm::AtomicOrdering::SequentiallyConsistent);
1059  break;
1060  }
1061  if (RValTy->isVoidType())
1062  return RValue::get(nullptr);
1063 
1064  return convertTempToRValue(
1065  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1066  RValTy, E->getExprLoc());
1067  }
1068 
1069  // Long case, when Order isn't obviously constant.
1070 
1071  // Create all the relevant BB's
1072  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1073  *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1074  *SeqCstBB = nullptr;
1075  MonotonicBB = createBasicBlock("monotonic", CurFn);
1076  if (!IsStore)
1077  AcquireBB = createBasicBlock("acquire", CurFn);
1078  if (!IsLoad)
1079  ReleaseBB = createBasicBlock("release", CurFn);
1080  if (!IsLoad && !IsStore)
1081  AcqRelBB = createBasicBlock("acqrel", CurFn);
1082  SeqCstBB = createBasicBlock("seqcst", CurFn);
1083  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1084 
1085  // Create the switch for the split
1086  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1087  // doesn't matter unless someone is crazy enough to use something that
1088  // doesn't fold to a constant for the ordering.
1089  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1090  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1091 
1092  // Emit all the different atomics
1093  Builder.SetInsertPoint(MonotonicBB);
1094  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1095  Size, llvm::AtomicOrdering::Monotonic);
1096  Builder.CreateBr(ContBB);
1097  if (!IsStore) {
1098  Builder.SetInsertPoint(AcquireBB);
1099  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1100  Size, llvm::AtomicOrdering::Acquire);
1101  Builder.CreateBr(ContBB);
1102  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1103  AcquireBB);
1104  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1105  AcquireBB);
1106  }
1107  if (!IsLoad) {
1108  Builder.SetInsertPoint(ReleaseBB);
1109  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1110  Size, llvm::AtomicOrdering::Release);
1111  Builder.CreateBr(ContBB);
1112  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1113  ReleaseBB);
1114  }
1115  if (!IsLoad && !IsStore) {
1116  Builder.SetInsertPoint(AcqRelBB);
1117  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1118  Size, llvm::AtomicOrdering::AcquireRelease);
1119  Builder.CreateBr(ContBB);
1120  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1121  AcqRelBB);
1122  }
1123  Builder.SetInsertPoint(SeqCstBB);
1124  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1125  Size, llvm::AtomicOrdering::SequentiallyConsistent);
1126  Builder.CreateBr(ContBB);
1127  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1128  SeqCstBB);
1129 
1130  // Cleanup and return
1131  Builder.SetInsertPoint(ContBB);
1132  if (RValTy->isVoidType())
1133  return RValue::get(nullptr);
1134 
1135  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1136  return convertTempToRValue(
1137  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1138  RValTy, E->getExprLoc());
1139 }
1140 
1141 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1142  unsigned addrspace =
1143  cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1144  llvm::IntegerType *ty =
1145  llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1146  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1147 }
1148 
1149 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1150  llvm::Type *Ty = Addr.getElementType();
1151  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1152  if (SourceSizeInBits != AtomicSizeInBits) {
1153  Address Tmp = CreateTempAlloca();
1154  CGF.Builder.CreateMemCpy(Tmp, Addr,
1155  std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1156  Addr = Tmp;
1157  }
1158 
1159  return emitCastToAtomicIntPointer(Addr);
1160 }
1161 
1162 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1163  AggValueSlot resultSlot,
1164  SourceLocation loc,
1165  bool asValue) const {
1166  if (LVal.isSimple()) {
1167  if (EvaluationKind == TEK_Aggregate)
1168  return resultSlot.asRValue();
1169 
1170  // Drill into the padding structure if we have one.
1171  if (hasPadding())
1172  addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1173 
1174  // Otherwise, just convert the temporary to an r-value using the
1175  // normal conversion routine.
1176  return CGF.convertTempToRValue(addr, getValueType(), loc);
1177  }
1178  if (!asValue)
1179  // Get RValue from temp memory as atomic for non-simple lvalues
1180  return RValue::get(CGF.Builder.CreateLoad(addr));
1181  if (LVal.isBitField())
1182  return CGF.EmitLoadOfBitfieldLValue(
1183  LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1184  LVal.getAlignmentSource()));
1185  if (LVal.isVectorElt())
1186  return CGF.EmitLoadOfLValue(
1187  LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1188  LVal.getAlignmentSource()), loc);
1189  assert(LVal.isExtVectorElt());
1191  addr, LVal.getExtVectorElts(), LVal.getType(),
1192  LVal.getAlignmentSource()));
1193 }
1194 
1195 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1196  AggValueSlot ResultSlot,
1197  SourceLocation Loc,
1198  bool AsValue) const {
1199  // Try not to in some easy cases.
1200  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1201  if (getEvaluationKind() == TEK_Scalar &&
1202  (((!LVal.isBitField() ||
1203  LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1204  !hasPadding()) ||
1205  !AsValue)) {
1206  auto *ValTy = AsValue
1207  ? CGF.ConvertTypeForMem(ValueTy)
1208  : getAtomicAddress().getType()->getPointerElementType();
1209  if (ValTy->isIntegerTy()) {
1210  assert(IntVal->getType() == ValTy && "Different integer types.");
1211  return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1212  } else if (ValTy->isPointerTy())
1213  return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1214  else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1215  return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1216  }
1217 
1218  // Create a temporary. This needs to be big enough to hold the
1219  // atomic integer.
1220  Address Temp = Address::invalid();
1221  bool TempIsVolatile = false;
1222  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1223  assert(!ResultSlot.isIgnored());
1224  Temp = ResultSlot.getAddress();
1225  TempIsVolatile = ResultSlot.isVolatile();
1226  } else {
1227  Temp = CreateTempAlloca();
1228  }
1229 
1230  // Slam the integer into the temporary.
1231  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1232  CGF.Builder.CreateStore(IntVal, CastTemp)
1233  ->setVolatile(TempIsVolatile);
1234 
1235  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1236 }
1237 
1238 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1239  llvm::AtomicOrdering AO, bool) {
1240  // void __atomic_load(size_t size, void *mem, void *return, int order);
1241  CallArgList Args;
1242  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1243  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1244  CGF.getContext().VoidPtrTy);
1245  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1246  CGF.getContext().VoidPtrTy);
1247  Args.add(
1248  RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1249  CGF.getContext().IntTy);
1250  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1251 }
1252 
1253 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1254  bool IsVolatile) {
1255  // Okay, we're doing this natively.
1256  Address Addr = getAtomicAddressAsAtomicIntPointer();
1257  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1258  Load->setAtomic(AO);
1259 
1260  // Other decoration.
1261  if (IsVolatile)
1262  Load->setVolatile(true);
1263  if (LVal.getTBAAInfo())
1264  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1265  return Load;
1266 }
1267 
1268 /// An LValue is a candidate for having its loads and stores be made atomic if
1269 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1270 /// performing such an operation can be performed without a libcall.
1272  if (!CGM.getCodeGenOpts().MSVolatile) return false;
1273  AtomicInfo AI(*this, LV);
1274  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1275  // An atomic is inline if we don't need to use a libcall.
1276  bool AtomicIsInline = !AI.shouldUseLibcall();
1277  // MSVC doesn't seem to do this for types wider than a pointer.
1278  if (getContext().getTypeSize(LV.getType()) >
1279  getContext().getTypeSize(getContext().getIntPtrType()))
1280  return false;
1281  return IsVolatile && AtomicIsInline;
1282 }
1283 
1285  AggValueSlot Slot) {
1286  llvm::AtomicOrdering AO;
1287  bool IsVolatile = LV.isVolatileQualified();
1288  if (LV.getType()->isAtomicType()) {
1289  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1290  } else {
1291  AO = llvm::AtomicOrdering::Acquire;
1292  IsVolatile = true;
1293  }
1294  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1295 }
1296 
1297 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1298  bool AsValue, llvm::AtomicOrdering AO,
1299  bool IsVolatile) {
1300  // Check whether we should use a library call.
1301  if (shouldUseLibcall()) {
1302  Address TempAddr = Address::invalid();
1303  if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1304  assert(getEvaluationKind() == TEK_Aggregate);
1305  TempAddr = ResultSlot.getAddress();
1306  } else
1307  TempAddr = CreateTempAlloca();
1308 
1309  EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1310 
1311  // Okay, turn that back into the original value or whole atomic (for
1312  // non-simple lvalues) type.
1313  return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1314  }
1315 
1316  // Okay, we're doing this natively.
1317  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1318 
1319  // If we're ignoring an aggregate return, don't do anything.
1320  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1321  return RValue::getAggregate(Address::invalid(), false);
1322 
1323  // Okay, turn that back into the original value or atomic (for non-simple
1324  // lvalues) type.
1325  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1326 }
1327 
1328 /// Emit a load from an l-value of atomic type. Note that the r-value
1329 /// we produce is an r-value of the atomic *value* type.
1331  llvm::AtomicOrdering AO, bool IsVolatile,
1332  AggValueSlot resultSlot) {
1333  AtomicInfo Atomics(*this, src);
1334  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1335  IsVolatile);
1336 }
1337 
1338 /// Copy an r-value into memory as part of storing to an atomic type.
1339 /// This needs to create a bit-pattern suitable for atomic operations.
1340 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1341  assert(LVal.isSimple());
1342  // If we have an r-value, the rvalue should be of the atomic type,
1343  // which means that the caller is responsible for having zeroed
1344  // any padding. Just do an aggregate copy of that type.
1345  if (rvalue.isAggregate()) {
1346  CGF.EmitAggregateCopy(getAtomicAddress(),
1347  rvalue.getAggregateAddress(),
1348  getAtomicType(),
1349  (rvalue.isVolatileQualified()
1350  || LVal.isVolatileQualified()));
1351  return;
1352  }
1353 
1354  // Okay, otherwise we're copying stuff.
1355 
1356  // Zero out the buffer if necessary.
1357  emitMemSetZeroIfNecessary();
1358 
1359  // Drill past the padding if present.
1360  LValue TempLVal = projectValue();
1361 
1362  // Okay, store the rvalue in.
1363  if (rvalue.isScalar()) {
1364  CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1365  } else {
1366  CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1367  }
1368 }
1369 
1370 
1371 /// Materialize an r-value into memory for the purposes of storing it
1372 /// to an atomic type.
1373 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1374  // Aggregate r-values are already in memory, and EmitAtomicStore
1375  // requires them to be values of the atomic type.
1376  if (rvalue.isAggregate())
1377  return rvalue.getAggregateAddress();
1378 
1379  // Otherwise, make a temporary and materialize into it.
1380  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1381  AtomicInfo Atomics(CGF, TempLV);
1382  Atomics.emitCopyIntoMemory(rvalue);
1383  return TempLV.getAddress();
1384 }
1385 
1386 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1387  // If we've got a scalar value of the right size, try to avoid going
1388  // through memory.
1389  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1390  llvm::Value *Value = RVal.getScalarVal();
1391  if (isa<llvm::IntegerType>(Value->getType()))
1392  return CGF.EmitToMemory(Value, ValueTy);
1393  else {
1394  llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1395  CGF.getLLVMContext(),
1396  LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1397  if (isa<llvm::PointerType>(Value->getType()))
1398  return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1399  else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1400  return CGF.Builder.CreateBitCast(Value, InputIntTy);
1401  }
1402  }
1403  // Otherwise, we need to go through memory.
1404  // Put the r-value in memory.
1405  Address Addr = materializeRValue(RVal);
1406 
1407  // Cast the temporary to the atomic int type and pull a value out.
1408  Addr = emitCastToAtomicIntPointer(Addr);
1409  return CGF.Builder.CreateLoad(Addr);
1410 }
1411 
1412 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1413  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1414  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1415  // Do the atomic store.
1416  Address Addr = getAtomicAddressAsAtomicIntPointer();
1417  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1418  ExpectedVal, DesiredVal,
1419  Success, Failure);
1420  // Other decoration.
1421  Inst->setVolatile(LVal.isVolatileQualified());
1422  Inst->setWeak(IsWeak);
1423 
1424  // Okay, turn that back into the original value type.
1425  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1426  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1427  return std::make_pair(PreviousVal, SuccessFailureVal);
1428 }
1429 
1430 llvm::Value *
1431 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1432  llvm::Value *DesiredAddr,
1433  llvm::AtomicOrdering Success,
1434  llvm::AtomicOrdering Failure) {
1435  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1436  // void *desired, int success, int failure);
1437  CallArgList Args;
1438  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1439  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1440  CGF.getContext().VoidPtrTy);
1441  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1442  CGF.getContext().VoidPtrTy);
1443  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1444  CGF.getContext().VoidPtrTy);
1445  Args.add(RValue::get(
1446  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1447  CGF.getContext().IntTy);
1448  Args.add(RValue::get(
1449  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1450  CGF.getContext().IntTy);
1451  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1452  CGF.getContext().BoolTy, Args);
1453 
1454  return SuccessFailureRVal.getScalarVal();
1455 }
1456 
1457 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1458  RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1459  llvm::AtomicOrdering Failure, bool IsWeak) {
1460  if (isStrongerThan(Failure, Success))
1461  // Don't assert on undefined behavior "failure argument shall be no stronger
1462  // than the success argument".
1463  Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1464 
1465  // Check whether we should use a library call.
1466  if (shouldUseLibcall()) {
1467  // Produce a source address.
1468  Address ExpectedAddr = materializeRValue(Expected);
1469  Address DesiredAddr = materializeRValue(Desired);
1470  auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1471  DesiredAddr.getPointer(),
1472  Success, Failure);
1473  return std::make_pair(
1474  convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1475  SourceLocation(), /*AsValue=*/false),
1476  Res);
1477  }
1478 
1479  // If we've got a scalar value of the right size, try to avoid going
1480  // through memory.
1481  auto *ExpectedVal = convertRValueToInt(Expected);
1482  auto *DesiredVal = convertRValueToInt(Desired);
1483  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1484  Failure, IsWeak);
1485  return std::make_pair(
1486  ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1487  SourceLocation(), /*AsValue=*/false),
1488  Res.second);
1489 }
1490 
1491 static void
1492 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1493  const llvm::function_ref<RValue(RValue)> &UpdateOp,
1494  Address DesiredAddr) {
1495  RValue UpRVal;
1496  LValue AtomicLVal = Atomics.getAtomicLValue();
1497  LValue DesiredLVal;
1498  if (AtomicLVal.isSimple()) {
1499  UpRVal = OldRVal;
1500  DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1501  } else {
1502  // Build new lvalue for temp address
1503  Address Ptr = Atomics.materializeRValue(OldRVal);
1504  LValue UpdateLVal;
1505  if (AtomicLVal.isBitField()) {
1506  UpdateLVal =
1507  LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1508  AtomicLVal.getType(),
1509  AtomicLVal.getAlignmentSource());
1510  DesiredLVal =
1511  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1512  AtomicLVal.getType(),
1513  AtomicLVal.getAlignmentSource());
1514  } else if (AtomicLVal.isVectorElt()) {
1515  UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1516  AtomicLVal.getType(),
1517  AtomicLVal.getAlignmentSource());
1518  DesiredLVal = LValue::MakeVectorElt(
1519  DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1520  AtomicLVal.getAlignmentSource());
1521  } else {
1522  assert(AtomicLVal.isExtVectorElt());
1523  UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1524  AtomicLVal.getType(),
1525  AtomicLVal.getAlignmentSource());
1526  DesiredLVal = LValue::MakeExtVectorElt(
1527  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1528  AtomicLVal.getAlignmentSource());
1529  }
1530  UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1531  DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1532  UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1533  }
1534  // Store new value in the corresponding memory area
1535  RValue NewRVal = UpdateOp(UpRVal);
1536  if (NewRVal.isScalar()) {
1537  CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1538  } else {
1539  assert(NewRVal.isComplex());
1540  CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1541  /*isInit=*/false);
1542  }
1543 }
1544 
1545 void AtomicInfo::EmitAtomicUpdateLibcall(
1546  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1547  bool IsVolatile) {
1548  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1549 
1550  Address ExpectedAddr = CreateTempAlloca();
1551 
1552  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1553  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1554  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1555  CGF.EmitBlock(ContBB);
1556  Address DesiredAddr = CreateTempAlloca();
1557  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1558  requiresMemSetZero(getAtomicAddress().getElementType())) {
1559  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1560  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1561  }
1562  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1564  SourceLocation(), /*AsValue=*/false);
1565  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1566  auto *Res =
1567  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1568  DesiredAddr.getPointer(),
1569  AO, Failure);
1570  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1571  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1572 }
1573 
1574 void AtomicInfo::EmitAtomicUpdateOp(
1575  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1576  bool IsVolatile) {
1577  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1578 
1579  // Do the atomic load.
1580  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1581  // For non-simple lvalues perform compare-and-swap procedure.
1582  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1583  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1584  auto *CurBB = CGF.Builder.GetInsertBlock();
1585  CGF.EmitBlock(ContBB);
1586  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1587  /*NumReservedValues=*/2);
1588  PHI->addIncoming(OldVal, CurBB);
1589  Address NewAtomicAddr = CreateTempAlloca();
1590  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1591  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1592  requiresMemSetZero(getAtomicAddress().getElementType())) {
1593  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1594  }
1595  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1596  SourceLocation(), /*AsValue=*/false);
1597  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1598  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1599  // Try to write new value using cmpxchg operation
1600  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1601  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1602  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1603  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1604 }
1605 
1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1607  RValue UpdateRVal, Address DesiredAddr) {
1608  LValue AtomicLVal = Atomics.getAtomicLValue();
1609  LValue DesiredLVal;
1610  // Build new lvalue for temp address
1611  if (AtomicLVal.isBitField()) {
1612  DesiredLVal =
1613  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1614  AtomicLVal.getType(),
1615  AtomicLVal.getAlignmentSource());
1616  } else if (AtomicLVal.isVectorElt()) {
1617  DesiredLVal =
1618  LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1619  AtomicLVal.getType(),
1620  AtomicLVal.getAlignmentSource());
1621  } else {
1622  assert(AtomicLVal.isExtVectorElt());
1623  DesiredLVal = LValue::MakeExtVectorElt(
1624  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1625  AtomicLVal.getAlignmentSource());
1626  }
1627  DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1628  // Store new value in the corresponding memory area
1629  assert(UpdateRVal.isScalar());
1630  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1631 }
1632 
1633 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1634  RValue UpdateRVal, bool IsVolatile) {
1635  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1636 
1637  Address ExpectedAddr = CreateTempAlloca();
1638 
1639  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1640  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1641  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1642  CGF.EmitBlock(ContBB);
1643  Address DesiredAddr = CreateTempAlloca();
1644  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1645  requiresMemSetZero(getAtomicAddress().getElementType())) {
1646  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1647  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1648  }
1649  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1650  auto *Res =
1651  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1652  DesiredAddr.getPointer(),
1653  AO, Failure);
1654  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1655  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1656 }
1657 
1658 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1659  bool IsVolatile) {
1660  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1661 
1662  // Do the atomic load.
1663  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1664  // For non-simple lvalues perform compare-and-swap procedure.
1665  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1666  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1667  auto *CurBB = CGF.Builder.GetInsertBlock();
1668  CGF.EmitBlock(ContBB);
1669  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1670  /*NumReservedValues=*/2);
1671  PHI->addIncoming(OldVal, CurBB);
1672  Address NewAtomicAddr = CreateTempAlloca();
1673  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1674  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1675  requiresMemSetZero(getAtomicAddress().getElementType())) {
1676  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1677  }
1678  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1679  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1680  // Try to write new value using cmpxchg operation
1681  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1682  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1683  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1684  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1685 }
1686 
1687 void AtomicInfo::EmitAtomicUpdate(
1688  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1689  bool IsVolatile) {
1690  if (shouldUseLibcall()) {
1691  EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1692  } else {
1693  EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1694  }
1695 }
1696 
1697 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1698  bool IsVolatile) {
1699  if (shouldUseLibcall()) {
1700  EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1701  } else {
1702  EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1703  }
1704 }
1705 
1707  bool isInit) {
1708  bool IsVolatile = lvalue.isVolatileQualified();
1709  llvm::AtomicOrdering AO;
1710  if (lvalue.getType()->isAtomicType()) {
1711  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1712  } else {
1713  AO = llvm::AtomicOrdering::Release;
1714  IsVolatile = true;
1715  }
1716  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1717 }
1718 
1719 /// Emit a store to an l-value of atomic type.
1720 ///
1721 /// Note that the r-value is expected to be an r-value *of the atomic
1722 /// type*; this means that for aggregate r-values, it should include
1723 /// storage for any padding that was necessary.
1725  llvm::AtomicOrdering AO, bool IsVolatile,
1726  bool isInit) {
1727  // If this is an aggregate r-value, it should agree in type except
1728  // maybe for address-space qualification.
1729  assert(!rvalue.isAggregate() ||
1731  == dest.getAddress().getElementType());
1732 
1733  AtomicInfo atomics(*this, dest);
1734  LValue LVal = atomics.getAtomicLValue();
1735 
1736  // If this is an initialization, just put the value there normally.
1737  if (LVal.isSimple()) {
1738  if (isInit) {
1739  atomics.emitCopyIntoMemory(rvalue);
1740  return;
1741  }
1742 
1743  // Check whether we should use a library call.
1744  if (atomics.shouldUseLibcall()) {
1745  // Produce a source address.
1746  Address srcAddr = atomics.materializeRValue(rvalue);
1747 
1748  // void __atomic_store(size_t size, void *mem, void *val, int order)
1749  CallArgList args;
1750  args.add(RValue::get(atomics.getAtomicSizeValue()),
1751  getContext().getSizeType());
1752  args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1753  getContext().VoidPtrTy);
1754  args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1755  getContext().VoidPtrTy);
1756  args.add(
1757  RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1758  getContext().IntTy);
1759  emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1760  return;
1761  }
1762 
1763  // Okay, we're doing this natively.
1764  llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1765 
1766  // Do the atomic store.
1767  Address addr =
1768  atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1769  intValue = Builder.CreateIntCast(
1770  intValue, addr.getElementType(), /*isSigned=*/false);
1771  llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1772 
1773  // Initializations don't need to be atomic.
1774  if (!isInit)
1775  store->setAtomic(AO);
1776 
1777  // Other decoration.
1778  if (IsVolatile)
1779  store->setVolatile(true);
1780  if (dest.getTBAAInfo())
1782  return;
1783  }
1784 
1785  // Emit simple atomic update operation.
1786  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1787 }
1788 
1789 /// Emit a compare-and-exchange op for atomic type.
1790 ///
1791 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1792  LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1793  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1794  AggValueSlot Slot) {
1795  // If this is an aggregate r-value, it should agree in type except
1796  // maybe for address-space qualification.
1797  assert(!Expected.isAggregate() ||
1798  Expected.getAggregateAddress().getElementType() ==
1799  Obj.getAddress().getElementType());
1800  assert(!Desired.isAggregate() ||
1801  Desired.getAggregateAddress().getElementType() ==
1802  Obj.getAddress().getElementType());
1803  AtomicInfo Atomics(*this, Obj);
1804 
1805  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1806  IsWeak);
1807 }
1808 
1810  LValue LVal, llvm::AtomicOrdering AO,
1811  const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1812  AtomicInfo Atomics(*this, LVal);
1813  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1814 }
1815 
1817  AtomicInfo atomics(*this, dest);
1818 
1819  switch (atomics.getEvaluationKind()) {
1820  case TEK_Scalar: {
1821  llvm::Value *value = EmitScalarExpr(init);
1822  atomics.emitCopyIntoMemory(RValue::get(value));
1823  return;
1824  }
1825 
1826  case TEK_Complex: {
1827  ComplexPairTy value = EmitComplexExpr(init);
1828  atomics.emitCopyIntoMemory(RValue::getComplex(value));
1829  return;
1830  }
1831 
1832  case TEK_Aggregate: {
1833  // Fix up the destination if the initializer isn't an expression
1834  // of atomic type.
1835  bool Zeroed = false;
1836  if (!init->getType()->isAtomicType()) {
1837  Zeroed = atomics.emitMemSetZeroIfNecessary();
1838  dest = atomics.projectValue();
1839  }
1840 
1841  // Evaluate the expression directly into the destination.
1846  Zeroed ? AggValueSlot::IsZeroed :
1848 
1849  EmitAggExpr(init, slot);
1850  return;
1851  }
1852  }
1853  llvm_unreachable("bad evaluation kind");
1854 }
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Definition: CGCall.h:151
Defines the clang::ASTContext interface.
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
Definition: CGAtomic.cpp:1791
llvm::IntegerType * IntTy
int
CanQualType VoidPtrTy
Definition: ASTContext.h:908
A (possibly-)qualified type.
Definition: Type.h:598
llvm::Type * ConvertTypeForMem(QualType T)
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, AlignmentSource alignSource)
Create a new object to represent a bit-field access.
Definition: CGValue.h:414
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, IsZeroed_t isZeroed=IsNotZeroed)
Definition: CGValue.h:524
void setAlignment(CharUnits A)
Definition: CGValue.h:317
const TargetInfo & getTarget() const
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:65
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
Address getAddress() const
Definition: CGValue.h:331
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void setTBAAInfo(llvm::MDNode *N)
Definition: CGValue.h:309
const llvm::DataLayout & getDataLayout() const
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
Definition: StoreRef.h:26
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1593
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Expr * getVal1() const
Definition: Expr.h:4840
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1706
bool isVolatile() const
Definition: CGValue.h:542
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
bool isVolatile() const
Definition: Expr.h:4869
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:1813
bool isVoidType() const
Definition: Type.h:5680
bool isVolatileQualified() const
Definition: CGValue.h:252
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:92
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size...
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Definition: CGAtomic.cpp:1809
Expr * getOrder() const
Definition: Expr.h:4837
RValue EmitCall(const CGFunctionInfo &FnInfo, llvm::Value *Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, CGCalleeInfo CalleeInfo=CGCalleeInfo(), llvm::Instruction **callOrInvoke=nullptr)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
Definition: CGCall.cpp:3507
RValue EmitLoadOfExtVectorElementLValue(LValue V)
Definition: CGExpr.cpp:1517
Expr * getVal2() const
Definition: Expr.h:4850
CharUnits getAlignment() const
Definition: CGValue.h:316
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
uint32_t Offset
Definition: CacheTokens.cpp:44
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:270
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1492
unsigned Align
Definition: ASTContext.h:83
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicInit(Expr *E, LValue lvalue)
Definition: CGAtomic.cpp:1816
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
RValue EmitAtomicExpr(AtomicExpr *E)
Definition: CGAtomic.cpp:661
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
bool isExtVectorElt() const
Definition: CGValue.h:249
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition: CGExpr.cpp:4156
bool isValid() const
Definition: Address.h:36
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, AlignmentSource alignSource)
Definition: CGValue.h:397
std::pair< CharUnits, CharUnits > getTypeInfoInChars(const Type *T) const
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:315
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource AlignSource=AlignmentSource::Type)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition: CGExpr.cpp:169
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:38
bool isCmpXChg() const
Definition: Expr.h:4873
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:415
static unsigned getNumSubExprs(AtomicOp Op)
Determine the number of arguments the specified atomic builtin should have.
Definition: Expr.cpp:3841
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, llvm::MDNode *TBAAInfo, bool ConvertTypeToTag=true)
Decorate the instruction with a TBAA tag.
static TypeEvaluationKind getEvaluationKind(QualType T)
hasAggregateLLVMType - Return true if the specified AST type will map into an aggregate LLVM type or ...
llvm::Value * getPointer() const
Definition: Address.h:38
Expr - This represents one expression.
Definition: Expr.h:105
static Address invalid()
Definition: Address.h:35
bool isAggregate() const
Definition: CGValue.h:53
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
Definition: CGExpr.cpp:1374
bool isAtomicType() const
Definition: Type.h:5564
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, AlignmentSource alignSource)
Definition: CGValue.h:386
RValue asRValue() const
Definition: CGValue.h:578
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
Definition: CGExpr.cpp:1349
bool isVectorElt() const
Definition: CGValue.h:247
static LValue MakeAddr(Address address, QualType type, ASTContext &Context, AlignmentSource alignSource, llvm::MDNode *TBAAInfo=nullptr)
Definition: CGValue.h:371
void add(RValue rvalue, QualType type, bool needscopy=false)
Definition: CGCall.h:81
char __ovld __cnfn min(char x, char y)
Returns y if y < x, otherwise it returns x.
llvm::LLVMContext & getLLVMContext()
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition: CGCall.cpp:586
Address EmitPointerWithAlignment(const Expr *Addr, AlignmentSource *Source=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition: CGExpr.cpp:820
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:292
Represents a GCC generic vector type.
Definition: Type.h:2756
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Definition: CGExpr.cpp:46
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order)
Definition: CGAtomic.cpp:488
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:405
The result type of a method or function.
AtomicOp getOp() const
Definition: Expr.h:4861
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1284
bool isVolatile() const
Definition: CGValue.h:295
The l-value was considered opaque, so the alignment was determined from a type.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:160
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeSet ExtraAttrs=llvm::AttributeSet())
Create a new runtime function with the specified type and name.
bool isSimple() const
Definition: CGValue.h:246
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:362
ASTContext & getContext() const
Encodes a location in the source.
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
Definition: CGAtomic.cpp:1271
Expr * getPtr() const
Definition: Expr.h:4834
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
Definition: CGExpr.cpp:1363
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:363
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Definition: CGExpr.cpp:1262
llvm::MDNode * getTBAAInfo() const
Definition: CGValue.h:308
An aggregate value slot.
Definition: CGValue.h:441
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load, __atomic_store, and __atomic_compare_exchange_*, for the similarly-named C++11 instructions, and __c11 variants for <stdatomic.h>.
Definition: Expr.h:4804
CanQualType VoidTy
Definition: ASTContext.h:893
const CodeGenOptions & getCodeGenOpts() const
An aligned address.
Definition: Address.h:25
bool isGlobalReg() const
Definition: CGValue.h:250
Address getExtVectorAddress() const
Definition: CGValue.h:346
bool isBitField() const
Definition: CGValue.h:248
AlignmentSource getAlignmentSource() const
Definition: CGValue.h:319
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:193
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
QualType getType() const
Definition: Expr.h:126
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
bool isScalar() const
Definition: CGValue.h:51
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:92
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:116
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
Definition: CGExpr.cpp:98
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition: ASTMatchers.h:1983
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:58
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:487
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
Definition: CGBuilder.h:183
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:70
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
detail::InMemoryDirectory::const_iterator E
void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, bool isAssignment=false)
EmitAggregateCopy - Emit an aggregate copy.
Definition: CGExprAgg.cpp:1459
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:113
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type. ...
Definition: CGExprAgg.cpp:1437
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:353
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:637
const T * getAs() const
Member-template getAs<specific type>'.
Definition: Type.h:5818
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
Address getAddress() const
Definition: CGValue.h:562
bool isComplex() const
Definition: CGValue.h:52
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:397
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type, returning the result.
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::Value * getVectorIdx() const
Definition: CGValue.h:343
Expr * getWeak() const
Definition: Expr.h:4856
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:5339
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:70
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1440
QualType getType() const
Definition: CGValue.h:258
uint64_t Width
Definition: ASTContext.h:82
CanQualType IntTy
Definition: ASTContext.h:901
RValue EmitLoadOfBitfieldLValue(LValue LV)
Definition: CGExpr.cpp:1486
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
static RValue get(llvm::Value *V)
Definition: CGValue.h:85
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder)
Definition: CGAtomic.cpp:357
bool isVolatileQualified() const
Definition: CGValue.h:55
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:303
#define true
Definition: stdbool.h:32
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:106
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
LValue - This represents an lvalue references.
Definition: CGValue.h:152
CanQualType BoolTy
Definition: ASTContext.h:894
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:175
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:56
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:629
Expr * getOrderFail() const
Definition: Expr.h:4846
bool isIgnored() const
Definition: CGValue.h:566
Structure with information about how a bitfield should be accessed.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:5286
bool isPointerType() const
Definition: Type.h:5482
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1466