LLVM 19.0.0git
AttributorAttributes.cpp
Go to the documentation of this file.
1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/Statistic.h"
40#include "llvm/IR/Argument.h"
41#include "llvm/IR/Assumptions.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/BasicBlock.h"
44#include "llvm/IR/Constant.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DataLayout.h"
48#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
50#include "llvm/IR/InlineAsm.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/IntrinsicsAMDGPU.h"
56#include "llvm/IR/IntrinsicsNVPTX.h"
57#include "llvm/IR/LLVMContext.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/NoFolder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueHandle.h"
74#include <cassert>
75#include <numeric>
76#include <optional>
77#include <string>
78
79using namespace llvm;
80
81#define DEBUG_TYPE "attributor"
82
84 "attributor-manifest-internal", cl::Hidden,
85 cl::desc("Manifest Attributor internal string attributes."),
86 cl::init(false));
87
88static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),
90
91template <>
93
95
97 "attributor-max-potential-values", cl::Hidden,
98 cl::desc("Maximum number of potential values to be "
99 "tracked for each position."),
101 cl::init(7));
102
104 "attributor-max-potential-values-iterations", cl::Hidden,
105 cl::desc(
106 "Maximum number of iterations we keep dismantling potential values."),
107 cl::init(64));
108
109STATISTIC(NumAAs, "Number of abstract attributes created");
110
111// Some helper macros to deal with statistics tracking.
112//
113// Usage:
114// For simple IR attribute tracking overload trackStatistics in the abstract
115// attribute and choose the right STATS_DECLTRACK_********* macro,
116// e.g.,:
117// void trackStatistics() const override {
118// STATS_DECLTRACK_ARG_ATTR(returned)
119// }
120// If there is a single "increment" side one can use the macro
121// STATS_DECLTRACK with a custom message. If there are multiple increment
122// sides, STATS_DECL and STATS_TRACK can also be used separately.
123//
124#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
125 ("Number of " #TYPE " marked '" #NAME "'")
126#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
127#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
128#define STATS_DECL(NAME, TYPE, MSG) \
129 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
130#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
131#define STATS_DECLTRACK(NAME, TYPE, MSG) \
132 { \
133 STATS_DECL(NAME, TYPE, MSG) \
134 STATS_TRACK(NAME, TYPE) \
135 }
136#define STATS_DECLTRACK_ARG_ATTR(NAME) \
137 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
138#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
139 STATS_DECLTRACK(NAME, CSArguments, \
140 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
141#define STATS_DECLTRACK_FN_ATTR(NAME) \
142 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
143#define STATS_DECLTRACK_CS_ATTR(NAME) \
144 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
145#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
146 STATS_DECLTRACK(NAME, FunctionReturn, \
147 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
148#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
149 STATS_DECLTRACK(NAME, CSReturn, \
150 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
151#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
152 STATS_DECLTRACK(NAME, Floating, \
153 ("Number of floating values known to be '" #NAME "'"))
154
155// Specialization of the operator<< for abstract attributes subclasses. This
156// disambiguates situations where multiple operators are applicable.
157namespace llvm {
158#define PIPE_OPERATOR(CLASS) \
159 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
160 return OS << static_cast<const AbstractAttribute &>(AA); \
161 }
162
200
201#undef PIPE_OPERATOR
202
203template <>
205 const DerefState &R) {
206 ChangeStatus CS0 =
207 clampStateAndIndicateChange(S.DerefBytesState, R.DerefBytesState);
208 ChangeStatus CS1 = clampStateAndIndicateChange(S.GlobalState, R.GlobalState);
209 return CS0 | CS1;
210}
211
212} // namespace llvm
213
214static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
215 bool HeaderOnly, Cycle **CPtr = nullptr) {
216 if (!CI)
217 return true;
218 auto *BB = I->getParent();
219 auto *C = CI->getCycle(BB);
220 if (!C)
221 return false;
222 if (CPtr)
223 *CPtr = C;
224 return !HeaderOnly || BB == C->getHeader();
225}
226
227/// Checks if a type could have padding bytes.
228static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
229 // There is no size information, so be conservative.
230 if (!Ty->isSized())
231 return false;
232
233 // If the alloc size is not equal to the storage size, then there are padding
234 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
235 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
236 return false;
237
238 // FIXME: This isn't the right way to check for padding in vectors with
239 // non-byte-size elements.
240 if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))
241 return isDenselyPacked(SeqTy->getElementType(), DL);
242
243 // For array types, check for padding within members.
244 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))
245 return isDenselyPacked(SeqTy->getElementType(), DL);
246
247 if (!isa<StructType>(Ty))
248 return true;
249
250 // Check for padding within and between elements of a struct.
251 StructType *StructTy = cast<StructType>(Ty);
252 const StructLayout *Layout = DL.getStructLayout(StructTy);
253 uint64_t StartPos = 0;
254 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
255 Type *ElTy = StructTy->getElementType(I);
256 if (!isDenselyPacked(ElTy, DL))
257 return false;
258 if (StartPos != Layout->getElementOffsetInBits(I))
259 return false;
260 StartPos += DL.getTypeAllocSizeInBits(ElTy);
261 }
262
263 return true;
264}
265
266/// Get pointer operand of memory accessing instruction. If \p I is
267/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
268/// is set to false and the instruction is volatile, return nullptr.
270 bool AllowVolatile) {
271 if (!AllowVolatile && I->isVolatile())
272 return nullptr;
273
274 if (auto *LI = dyn_cast<LoadInst>(I)) {
275 return LI->getPointerOperand();
276 }
277
278 if (auto *SI = dyn_cast<StoreInst>(I)) {
279 return SI->getPointerOperand();
280 }
281
282 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) {
283 return CXI->getPointerOperand();
284 }
285
286 if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
287 return RMWI->getPointerOperand();
288 }
289
290 return nullptr;
291}
292
293/// Helper function to create a pointer based on \p Ptr, and advanced by \p
294/// Offset bytes.
296 IRBuilder<NoFolder> &IRB) {
297 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
298 << "-bytes\n");
299
300 if (Offset)
301 Ptr = IRB.CreatePtrAdd(Ptr, IRB.getInt64(Offset),
302 Ptr->getName() + ".b" + Twine(Offset));
303 return Ptr;
304}
305
306static const Value *
308 const Value *Val, const DataLayout &DL, APInt &Offset,
309 bool GetMinOffset, bool AllowNonInbounds,
310 bool UseAssumed = false) {
311
312 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
313 const IRPosition &Pos = IRPosition::value(V);
314 // Only track dependence if we are going to use the assumed info.
315 const AAValueConstantRange *ValueConstantRangeAA =
316 A.getAAFor<AAValueConstantRange>(QueryingAA, Pos,
317 UseAssumed ? DepClassTy::OPTIONAL
318 : DepClassTy::NONE);
319 if (!ValueConstantRangeAA)
320 return false;
321 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
322 : ValueConstantRangeAA->getKnown();
323 if (Range.isFullSet())
324 return false;
325
326 // We can only use the lower part of the range because the upper part can
327 // be higher than what the value can really be.
328 if (GetMinOffset)
329 ROffset = Range.getSignedMin();
330 else
331 ROffset = Range.getSignedMax();
332 return true;
333 };
334
335 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
336 /* AllowInvariant */ true,
337 AttributorAnalysis);
338}
339
340static const Value *
342 const Value *Ptr, int64_t &BytesOffset,
343 const DataLayout &DL, bool AllowNonInbounds = false) {
344 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
345 const Value *Base =
346 stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt,
347 /* GetMinOffset */ true, AllowNonInbounds);
348
349 BytesOffset = OffsetAPInt.getSExtValue();
350 return Base;
351}
352
353/// Clamp the information known for all returned values of a function
354/// (identified by \p QueryingAA) into \p S.
355template <typename AAType, typename StateType = typename AAType::StateType,
356 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
357 bool RecurseForSelectAndPHI = true>
359 Attributor &A, const AAType &QueryingAA, StateType &S,
360 const IRPosition::CallBaseContext *CBContext = nullptr) {
361 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
362 << QueryingAA << " into " << S << "\n");
363
364 assert((QueryingAA.getIRPosition().getPositionKind() ==
366 QueryingAA.getIRPosition().getPositionKind() ==
368 "Can only clamp returned value states for a function returned or call "
369 "site returned position!");
370
371 // Use an optional state as there might not be any return values and we want
372 // to join (IntegerState::operator&) the state of all there are.
373 std::optional<StateType> T;
374
375 // Callback for each possibly returned value.
376 auto CheckReturnValue = [&](Value &RV) -> bool {
377 const IRPosition &RVPos = IRPosition::value(RV, CBContext);
378 // If possible, use the hasAssumedIRAttr interface.
379 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
380 bool IsKnown;
381 return AA::hasAssumedIRAttr<IRAttributeKind>(
382 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
383 }
384
385 const AAType *AA =
386 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
387 if (!AA)
388 return false;
389 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
390 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
391 const StateType &AAS = AA->getState();
392 if (!T)
393 T = StateType::getBestState(AAS);
394 *T &= AAS;
395 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
396 << "\n");
397 return T->isValidState();
398 };
399
400 if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,
401 AA::ValueScope::Intraprocedural,
402 RecurseForSelectAndPHI))
403 S.indicatePessimisticFixpoint();
404 else if (T)
405 S ^= *T;
406}
407
408namespace {
409/// Helper class for generic deduction: return value -> returned position.
410template <typename AAType, typename BaseType,
411 typename StateType = typename BaseType::StateType,
412 bool PropagateCallBaseContext = false,
413 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
414 bool RecurseForSelectAndPHI = true>
415struct AAReturnedFromReturnedValues : public BaseType {
416 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
417 : BaseType(IRP, A) {}
418
419 /// See AbstractAttribute::updateImpl(...).
420 ChangeStatus updateImpl(Attributor &A) override {
421 StateType S(StateType::getBestState(this->getState()));
422 clampReturnedValueStates<AAType, StateType, IRAttributeKind, RecurseForSelectAndPHI>(
423 A, *this, S,
424 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
425 // TODO: If we know we visited all returned values, thus no are assumed
426 // dead, we can take the known information from the state T.
427 return clampStateAndIndicateChange<StateType>(this->getState(), S);
428 }
429};
430
431/// Clamp the information known at all call sites for a given argument
432/// (identified by \p QueryingAA) into \p S.
433template <typename AAType, typename StateType = typename AAType::StateType,
434 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
435static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
436 StateType &S) {
437 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
438 << QueryingAA << " into " << S << "\n");
439
440 assert(QueryingAA.getIRPosition().getPositionKind() ==
442 "Can only clamp call site argument states for an argument position!");
443
444 // Use an optional state as there might not be any return values and we want
445 // to join (IntegerState::operator&) the state of all there are.
446 std::optional<StateType> T;
447
448 // The argument number which is also the call site argument number.
449 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
450
451 auto CallSiteCheck = [&](AbstractCallSite ACS) {
452 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
453 // Check if a coresponding argument was found or if it is on not associated
454 // (which can happen for callback calls).
455 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
456 return false;
457
458 // If possible, use the hasAssumedIRAttr interface.
459 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
460 bool IsKnown;
461 return AA::hasAssumedIRAttr<IRAttributeKind>(
462 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
463 }
464
465 const AAType *AA =
466 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
467 if (!AA)
468 return false;
469 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
470 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
471 << "\n");
472 const StateType &AAS = AA->getState();
473 if (!T)
474 T = StateType::getBestState(AAS);
475 *T &= AAS;
476 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
477 << "\n");
478 return T->isValidState();
479 };
480
481 bool UsedAssumedInformation = false;
482 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
483 UsedAssumedInformation))
484 S.indicatePessimisticFixpoint();
485 else if (T)
486 S ^= *T;
487}
488
489/// This function is the bridge between argument position and the call base
490/// context.
491template <typename AAType, typename BaseType,
492 typename StateType = typename AAType::StateType,
493 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
494bool getArgumentStateFromCallBaseContext(Attributor &A,
495 BaseType &QueryingAttribute,
496 IRPosition &Pos, StateType &State) {
498 "Expected an 'argument' position !");
499 const CallBase *CBContext = Pos.getCallBaseContext();
500 if (!CBContext)
501 return false;
502
503 int ArgNo = Pos.getCallSiteArgNo();
504 assert(ArgNo >= 0 && "Invalid Arg No!");
505 const IRPosition CBArgPos = IRPosition::callsite_argument(*CBContext, ArgNo);
506
507 // If possible, use the hasAssumedIRAttr interface.
508 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
509 bool IsKnown;
510 return AA::hasAssumedIRAttr<IRAttributeKind>(
511 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
512 }
513
514 const auto *AA =
515 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
516 if (!AA)
517 return false;
518 const StateType &CBArgumentState =
519 static_cast<const StateType &>(AA->getState());
520
521 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
522 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
523 << "\n");
524
525 // NOTE: If we want to do call site grouping it should happen here.
526 State ^= CBArgumentState;
527 return true;
528}
529
530/// Helper class for generic deduction: call site argument -> argument position.
531template <typename AAType, typename BaseType,
532 typename StateType = typename AAType::StateType,
533 bool BridgeCallBaseContext = false,
534 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
535struct AAArgumentFromCallSiteArguments : public BaseType {
536 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
537 : BaseType(IRP, A) {}
538
539 /// See AbstractAttribute::updateImpl(...).
540 ChangeStatus updateImpl(Attributor &A) override {
541 StateType S = StateType::getBestState(this->getState());
542
543 if (BridgeCallBaseContext) {
544 bool Success =
545 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
546 IRAttributeKind>(
547 A, *this, this->getIRPosition(), S);
548 if (Success)
549 return clampStateAndIndicateChange<StateType>(this->getState(), S);
550 }
551 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
552 S);
553
554 // TODO: If we know we visited all incoming values, thus no are assumed
555 // dead, we can take the known information from the state T.
556 return clampStateAndIndicateChange<StateType>(this->getState(), S);
557 }
558};
559
560/// Helper class for generic replication: function returned -> cs returned.
561template <typename AAType, typename BaseType,
562 typename StateType = typename BaseType::StateType,
563 bool IntroduceCallBaseContext = false,
564 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
565struct AACalleeToCallSite : public BaseType {
566 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
567
568 /// See AbstractAttribute::updateImpl(...).
569 ChangeStatus updateImpl(Attributor &A) override {
570 auto IRPKind = this->getIRPosition().getPositionKind();
572 IRPKind == IRPosition::IRP_CALL_SITE) &&
573 "Can only wrap function returned positions for call site "
574 "returned positions!");
575 auto &S = this->getState();
576
577 CallBase &CB = cast<CallBase>(this->getAnchorValue());
578 if (IntroduceCallBaseContext)
579 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
580 << "\n");
581
582 ChangeStatus Changed = ChangeStatus::UNCHANGED;
583 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
584 for (const Function *Callee : Callees) {
585 IRPosition FnPos =
587 ? IRPosition::returned(*Callee,
588 IntroduceCallBaseContext ? &CB : nullptr)
590 *Callee, IntroduceCallBaseContext ? &CB : nullptr);
591 // If possible, use the hasAssumedIRAttr interface.
592 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
593 bool IsKnown;
594 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
595 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
596 return false;
597 continue;
598 }
599
600 const AAType *AA =
601 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
602 if (!AA)
603 return false;
604 Changed |= clampStateAndIndicateChange(S, AA->getState());
605 if (S.isAtFixpoint())
606 return S.isValidState();
607 }
608 return true;
609 };
610 if (!A.checkForAllCallees(CalleePred, *this, CB))
611 return S.indicatePessimisticFixpoint();
612 return Changed;
613 }
614};
615
616/// Helper function to accumulate uses.
617template <class AAType, typename StateType = typename AAType::StateType>
618static void followUsesInContext(AAType &AA, Attributor &A,
620 const Instruction *CtxI,
622 StateType &State) {
623 auto EIt = Explorer.begin(CtxI), EEnd = Explorer.end(CtxI);
624 for (unsigned u = 0; u < Uses.size(); ++u) {
625 const Use *U = Uses[u];
626 if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
627 bool Found = Explorer.findInContextOf(UserI, EIt, EEnd);
628 if (Found && AA.followUseInMBEC(A, U, UserI, State))
629 for (const Use &Us : UserI->uses())
630 Uses.insert(&Us);
631 }
632 }
633}
634
635/// Use the must-be-executed-context around \p I to add information into \p S.
636/// The AAType class is required to have `followUseInMBEC` method with the
637/// following signature and behaviour:
638///
639/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
640/// U - Underlying use.
641/// I - The user of the \p U.
642/// Returns true if the value should be tracked transitively.
643///
644template <class AAType, typename StateType = typename AAType::StateType>
645static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
646 Instruction &CtxI) {
648 A.getInfoCache().getMustBeExecutedContextExplorer();
649 if (!Explorer)
650 return;
651
652 // Container for (transitive) uses of the associated value.
654 for (const Use &U : AA.getIRPosition().getAssociatedValue().uses())
655 Uses.insert(&U);
656
657 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
658
659 if (S.isAtFixpoint())
660 return;
661
663 auto Pred = [&](const Instruction *I) {
664 if (const BranchInst *Br = dyn_cast<BranchInst>(I))
665 if (Br->isConditional())
666 BrInsts.push_back(Br);
667 return true;
668 };
669
670 // Here, accumulate conditional branch instructions in the context. We
671 // explore the child paths and collect the known states. The disjunction of
672 // those states can be merged to its own state. Let ParentState_i be a state
673 // to indicate the known information for an i-th branch instruction in the
674 // context. ChildStates are created for its successors respectively.
675 //
676 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
677 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
678 // ...
679 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
680 //
681 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
682 //
683 // FIXME: Currently, recursive branches are not handled. For example, we
684 // can't deduce that ptr must be dereferenced in below function.
685 //
686 // void f(int a, int c, int *ptr) {
687 // if(a)
688 // if (b) {
689 // *ptr = 0;
690 // } else {
691 // *ptr = 1;
692 // }
693 // else {
694 // if (b) {
695 // *ptr = 0;
696 // } else {
697 // *ptr = 1;
698 // }
699 // }
700 // }
701
702 Explorer->checkForAllContext(&CtxI, Pred);
703 for (const BranchInst *Br : BrInsts) {
704 StateType ParentState;
705
706 // The known state of the parent state is a conjunction of children's
707 // known states so it is initialized with a best state.
708 ParentState.indicateOptimisticFixpoint();
709
710 for (const BasicBlock *BB : Br->successors()) {
711 StateType ChildState;
712
713 size_t BeforeSize = Uses.size();
714 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
715
716 // Erase uses which only appear in the child.
717 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
718 It = Uses.erase(It);
719
720 ParentState &= ChildState;
721 }
722
723 // Use only known state.
724 S += ParentState;
725 }
726}
727} // namespace
728
729/// ------------------------ PointerInfo ---------------------------------------
730
731namespace llvm {
732namespace AA {
733namespace PointerInfo {
734
735struct State;
736
737} // namespace PointerInfo
738} // namespace AA
739
740/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
741template <>
744 static inline Access getEmptyKey();
745 static inline Access getTombstoneKey();
746 static unsigned getHashValue(const Access &A);
747 static bool isEqual(const Access &LHS, const Access &RHS);
748};
749
750/// Helper that allows RangeTy as a key in a DenseMap.
751template <> struct DenseMapInfo<AA::RangeTy> {
752 static inline AA::RangeTy getEmptyKey() {
753 auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
754 return AA::RangeTy{EmptyKey, EmptyKey};
755 }
756
757 static inline AA::RangeTy getTombstoneKey() {
758 auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
759 return AA::RangeTy{TombstoneKey, TombstoneKey};
760 }
761
762 static unsigned getHashValue(const AA::RangeTy &Range) {
766 }
767
768 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
769 return A == B;
770 }
771};
772
773/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
774/// but the instruction
775struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
778 static inline Access getEmptyKey();
779 static inline Access getTombstoneKey();
780 static unsigned getHashValue(const Access &A);
781 static bool isEqual(const Access &LHS, const Access &RHS);
782};
783
784} // namespace llvm
785
786/// A type to track pointer/struct usage and accesses for AAPointerInfo.
788 /// Return the best possible representable state.
789 static State getBestState(const State &SIS) { return State(); }
790
791 /// Return the worst possible representable state.
792 static State getWorstState(const State &SIS) {
793 State R;
794 R.indicatePessimisticFixpoint();
795 return R;
796 }
797
798 State() = default;
799 State(State &&SIS) = default;
800
801 const State &getAssumed() const { return *this; }
802
803 /// See AbstractState::isValidState().
804 bool isValidState() const override { return BS.isValidState(); }
805
806 /// See AbstractState::isAtFixpoint().
807 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
808
809 /// See AbstractState::indicateOptimisticFixpoint().
813 }
814
815 /// See AbstractState::indicatePessimisticFixpoint().
819 }
820
821 State &operator=(const State &R) {
822 if (this == &R)
823 return *this;
824 BS = R.BS;
825 AccessList = R.AccessList;
826 OffsetBins = R.OffsetBins;
827 RemoteIMap = R.RemoteIMap;
828 return *this;
829 }
830
832 if (this == &R)
833 return *this;
834 std::swap(BS, R.BS);
835 std::swap(AccessList, R.AccessList);
836 std::swap(OffsetBins, R.OffsetBins);
837 std::swap(RemoteIMap, R.RemoteIMap);
838 return *this;
839 }
840
841 /// Add a new Access to the state at offset \p Offset and with size \p Size.
842 /// The access is associated with \p I, writes \p Content (if anything), and
843 /// is of kind \p Kind. If an Access already exists for the same \p I and same
844 /// \p RemoteI, the two are combined, potentially losing information about
845 /// offset and size. The resulting access must now be moved from its original
846 /// OffsetBin to the bin for its new offset.
847 ///
848 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
850 Instruction &I, std::optional<Value *> Content,
852 Instruction *RemoteI = nullptr);
853
856 int64_t numOffsetBins() const { return OffsetBins.size(); }
857
858 const AAPointerInfo::Access &getAccess(unsigned Index) const {
859 return AccessList[Index];
860 }
861
862protected:
863 // Every memory instruction results in an Access object. We maintain a list of
864 // all Access objects that we own, along with the following maps:
865 //
866 // - OffsetBins: RangeTy -> { Access }
867 // - RemoteIMap: RemoteI x LocalI -> Access
868 //
869 // A RemoteI is any instruction that accesses memory. RemoteI is different
870 // from LocalI if and only if LocalI is a call; then RemoteI is some
871 // instruction in the callgraph starting from LocalI. Multiple paths in the
872 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
873 // are all combined into a single Access object. This may result in loss of
874 // information in RangeTy in the Access object.
878
879 /// See AAPointerInfo::forallInterferingAccesses.
881 AA::RangeTy Range,
882 function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const {
883 if (!isValidState())
884 return false;
885
886 for (const auto &It : OffsetBins) {
887 AA::RangeTy ItRange = It.getFirst();
888 if (!Range.mayOverlap(ItRange))
889 continue;
890 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
891 for (auto Index : It.getSecond()) {
892 auto &Access = AccessList[Index];
893 if (!CB(Access, IsExact))
894 return false;
895 }
896 }
897 return true;
898 }
899
900 /// See AAPointerInfo::forallInterferingAccesses.
902 Instruction &I,
903 function_ref<bool(const AAPointerInfo::Access &, bool)> CB,
904 AA::RangeTy &Range) const {
905 if (!isValidState())
906 return false;
907
908 auto LocalList = RemoteIMap.find(&I);
909 if (LocalList == RemoteIMap.end()) {
910 return true;
911 }
912
913 for (unsigned Index : LocalList->getSecond()) {
914 for (auto &R : AccessList[Index]) {
915 Range &= R;
916 if (Range.offsetAndSizeAreUnknown())
917 break;
918 }
919 }
920 return forallInterferingAccesses(Range, CB);
921 }
922
923private:
924 /// State to track fixpoint and validity.
925 BooleanState BS;
926};
927
930 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
931 Instruction *RemoteI) {
932 RemoteI = RemoteI ? RemoteI : &I;
933
934 // Check if we have an access for this instruction, if not, simply add it.
935 auto &LocalList = RemoteIMap[RemoteI];
936 bool AccExists = false;
937 unsigned AccIndex = AccessList.size();
938 for (auto Index : LocalList) {
939 auto &A = AccessList[Index];
940 if (A.getLocalInst() == &I) {
941 AccExists = true;
942 AccIndex = Index;
943 break;
944 }
945 }
946
947 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
948 LLVM_DEBUG(if (ToAdd.size()) dbgs()
949 << "[AAPointerInfo] Inserting access in new offset bins\n";);
950
951 for (auto Key : ToAdd) {
952 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
953 OffsetBins[Key].insert(AccIndex);
954 }
955 };
956
957 if (!AccExists) {
958 AccessList.emplace_back(&I, RemoteI, Ranges, Content, Kind, Ty);
959 assert((AccessList.size() == AccIndex + 1) &&
960 "New Access should have been at AccIndex");
961 LocalList.push_back(AccIndex);
962 AddToBins(AccessList[AccIndex].getRanges());
964 }
965
966 // Combine the new Access with the existing Access, and then update the
967 // mapping in the offset bins.
968 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
969 auto &Current = AccessList[AccIndex];
970 auto Before = Current;
971 Current &= Acc;
972 if (Current == Before)
974
975 auto &ExistingRanges = Before.getRanges();
976 auto &NewRanges = Current.getRanges();
977
978 // Ranges that are in the old access but not the new access need to be removed
979 // from the offset bins.
981 AAPointerInfo::RangeList::set_difference(ExistingRanges, NewRanges, ToRemove);
982 LLVM_DEBUG(if (ToRemove.size()) dbgs()
983 << "[AAPointerInfo] Removing access from old offset bins\n";);
984
985 for (auto Key : ToRemove) {
986 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
987 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
988 auto &Bin = OffsetBins[Key];
989 assert(Bin.count(AccIndex) &&
990 "Expected bin to actually contain the Access.");
991 Bin.erase(AccIndex);
992 }
993
994 // Ranges that are in the new access but not the old access need to be added
995 // to the offset bins.
997 AAPointerInfo::RangeList::set_difference(NewRanges, ExistingRanges, ToAdd);
998 AddToBins(ToAdd);
1000}
1001
1002namespace {
1003
1004/// A helper containing a list of offsets computed for a Use. Ideally this
1005/// list should be strictly ascending, but we ensure that only when we
1006/// actually translate the list of offsets to a RangeList.
1007struct OffsetInfo {
1008 using VecTy = SmallVector<int64_t>;
1009 using const_iterator = VecTy::const_iterator;
1010 VecTy Offsets;
1011
1012 const_iterator begin() const { return Offsets.begin(); }
1013 const_iterator end() const { return Offsets.end(); }
1014
1015 bool operator==(const OffsetInfo &RHS) const {
1016 return Offsets == RHS.Offsets;
1017 }
1018
1019 bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
1020
1021 void insert(int64_t Offset) { Offsets.push_back(Offset); }
1022 bool isUnassigned() const { return Offsets.size() == 0; }
1023
1024 bool isUnknown() const {
1025 if (isUnassigned())
1026 return false;
1027 if (Offsets.size() == 1)
1028 return Offsets.front() == AA::RangeTy::Unknown;
1029 return false;
1030 }
1031
1032 void setUnknown() {
1033 Offsets.clear();
1034 Offsets.push_back(AA::RangeTy::Unknown);
1035 }
1036
1037 void addToAll(int64_t Inc) {
1038 for (auto &Offset : Offsets) {
1039 Offset += Inc;
1040 }
1041 }
1042
1043 /// Copy offsets from \p R into the current list.
1044 ///
1045 /// Ideally all lists should be strictly ascending, but we defer that to the
1046 /// actual use of the list. So we just blindly append here.
1047 void merge(const OffsetInfo &R) { Offsets.append(R.Offsets); }
1048};
1049
1050#ifndef NDEBUG
1051static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) {
1052 ListSeparator LS;
1053 OS << "[";
1054 for (auto Offset : OI) {
1055 OS << LS << Offset;
1056 }
1057 OS << "]";
1058 return OS;
1059}
1060#endif // NDEBUG
1061
1062struct AAPointerInfoImpl
1063 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1065 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1066
1067 /// See AbstractAttribute::getAsStr().
1068 const std::string getAsStr(Attributor *A) const override {
1069 return std::string("PointerInfo ") +
1070 (isValidState() ? (std::string("#") +
1071 std::to_string(OffsetBins.size()) + " bins")
1072 : "<invalid>");
1073 }
1074
1075 /// See AbstractAttribute::manifest(...).
1076 ChangeStatus manifest(Attributor &A) override {
1077 return AAPointerInfo::manifest(A);
1078 }
1079
1080 virtual const_bin_iterator begin() const override { return State::begin(); }
1081 virtual const_bin_iterator end() const override { return State::end(); }
1082 virtual int64_t numOffsetBins() const override {
1083 return State::numOffsetBins();
1084 }
1085
1086 bool forallInterferingAccesses(
1087 AA::RangeTy Range,
1088 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1089 const override {
1090 return State::forallInterferingAccesses(Range, CB);
1091 }
1092
1093 bool forallInterferingAccesses(
1094 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1095 bool FindInterferingWrites, bool FindInterferingReads,
1096 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1097 AA::RangeTy &Range,
1098 function_ref<bool(const Access &)> SkipCB) const override {
1099 HasBeenWrittenTo = false;
1100
1101 SmallPtrSet<const Access *, 8> DominatingWrites;
1102 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1103
1104 Function &Scope = *I.getFunction();
1105 bool IsKnownNoSync;
1106 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1107 A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1108 IsKnownNoSync);
1109 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1110 IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE);
1111 bool AllInSameNoSyncFn = IsAssumedNoSync;
1112 bool InstIsExecutedByInitialThreadOnly =
1113 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1114
1115 // If the function is not ending in aligned barriers, we need the stores to
1116 // be in aligned barriers. The load being in one is not sufficient since the
1117 // store might be executed by a thread that disappears after, causing the
1118 // aligned barrier guarding the load to unblock and the load to read a value
1119 // that has no CFG path to the load.
1120 bool InstIsExecutedInAlignedRegion =
1121 FindInterferingReads && ExecDomainAA &&
1122 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1123
1124 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1125 A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1126
1127 InformationCache &InfoCache = A.getInfoCache();
1128 bool IsThreadLocalObj =
1129 AA::isAssumedThreadLocalObject(A, getAssociatedValue(), *this);
1130
1131 // Helper to determine if we need to consider threading, which we cannot
1132 // right now. However, if the function is (assumed) nosync or the thread
1133 // executing all instructions is the main thread only we can ignore
1134 // threading. Also, thread-local objects do not require threading reasoning.
1135 // Finally, we can ignore threading if either access is executed in an
1136 // aligned region.
1137 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1138 if (IsThreadLocalObj || AllInSameNoSyncFn)
1139 return true;
1140 const auto *FnExecDomainAA =
1141 I.getFunction() == &Scope
1142 ? ExecDomainAA
1143 : A.lookupAAFor<AAExecutionDomain>(
1144 IRPosition::function(*I.getFunction()), &QueryingAA,
1145 DepClassTy::NONE);
1146 if (!FnExecDomainAA)
1147 return false;
1148 if (InstIsExecutedInAlignedRegion ||
1149 (FindInterferingWrites &&
1150 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1151 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1152 return true;
1153 }
1154 if (InstIsExecutedByInitialThreadOnly &&
1155 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1156 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1157 return true;
1158 }
1159 return false;
1160 };
1161
1162 // Helper to determine if the access is executed by the same thread as the
1163 // given instruction, for now it is sufficient to avoid any potential
1164 // threading effects as we cannot deal with them anyway.
1165 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1166 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1167 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1168 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1169 };
1170
1171 // TODO: Use inter-procedural reachability and dominance.
1172 bool IsKnownNoRecurse;
1173 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1174 A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1175 IsKnownNoRecurse);
1176
1177 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1178 // AAExecutionDomain) such that we allow scopes other than kernels as long
1179 // as the reaching kernels are disjoint.
1180 bool InstInKernel = Scope.hasFnAttribute("kernel");
1181 bool ObjHasKernelLifetime = false;
1182 const bool UseDominanceReasoning =
1183 FindInterferingWrites && IsKnownNoRecurse;
1184 const DominatorTree *DT =
1186
1187 // Helper to check if a value has "kernel lifetime", that is it will not
1188 // outlive a GPU kernel. This is true for shared, constant, and local
1189 // globals on AMD and NVIDIA GPUs.
1190 auto HasKernelLifetime = [&](Value *V, Module &M) {
1191 if (!AA::isGPU(M))
1192 return false;
1193 switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
1194 case AA::GPUAddressSpace::Shared:
1195 case AA::GPUAddressSpace::Constant:
1196 case AA::GPUAddressSpace::Local:
1197 return true;
1198 default:
1199 return false;
1200 };
1201 };
1202
1203 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1204 // to determine if we should look at reachability from the callee. For
1205 // certain pointers we know the lifetime and we do not have to step into the
1206 // callee to determine reachability as the pointer would be dead in the
1207 // callee. See the conditional initialization below.
1208 std::function<bool(const Function &)> IsLiveInCalleeCB;
1209
1210 if (auto *AI = dyn_cast<AllocaInst>(&getAssociatedValue())) {
1211 // If the alloca containing function is not recursive the alloca
1212 // must be dead in the callee.
1213 const Function *AIFn = AI->getFunction();
1214 ObjHasKernelLifetime = AIFn->hasFnAttribute("kernel");
1215 bool IsKnownNoRecurse;
1216 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1217 A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
1218 IsKnownNoRecurse)) {
1219 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1220 }
1221 } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {
1222 // If the global has kernel lifetime we can stop if we reach a kernel
1223 // as it is "dead" in the (unknown) callees.
1224 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1225 if (ObjHasKernelLifetime)
1226 IsLiveInCalleeCB = [](const Function &Fn) {
1227 return !Fn.hasFnAttribute("kernel");
1228 };
1229 }
1230
1231 // Set of accesses/instructions that will overwrite the result and are
1232 // therefore blockers in the reachability traversal.
1233 AA::InstExclusionSetTy ExclusionSet;
1234
1235 auto AccessCB = [&](const Access &Acc, bool Exact) {
1236 Function *AccScope = Acc.getRemoteInst()->getFunction();
1237 bool AccInSameScope = AccScope == &Scope;
1238
1239 // If the object has kernel lifetime we can ignore accesses only reachable
1240 // by other kernels. For now we only skip accesses *in* other kernels.
1241 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1242 AccScope->hasFnAttribute("kernel"))
1243 return true;
1244
1245 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1246 if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))
1247 ExclusionSet.insert(Acc.getRemoteInst());
1248 }
1249
1250 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1251 (!FindInterferingReads || !Acc.isRead()))
1252 return true;
1253
1254 bool Dominates = FindInterferingWrites && DT && Exact &&
1255 Acc.isMustAccess() && AccInSameScope &&
1256 DT->dominates(Acc.getRemoteInst(), &I);
1257 if (Dominates)
1258 DominatingWrites.insert(&Acc);
1259
1260 // Track if all interesting accesses are in the same `nosync` function as
1261 // the given instruction.
1262 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1263
1264 InterferingAccesses.push_back({&Acc, Exact});
1265 return true;
1266 };
1267 if (!State::forallInterferingAccesses(I, AccessCB, Range))
1268 return false;
1269
1270 HasBeenWrittenTo = !DominatingWrites.empty();
1271
1272 // Dominating writes form a chain, find the least/lowest member.
1273 Instruction *LeastDominatingWriteInst = nullptr;
1274 for (const Access *Acc : DominatingWrites) {
1275 if (!LeastDominatingWriteInst) {
1276 LeastDominatingWriteInst = Acc->getRemoteInst();
1277 } else if (DT->dominates(LeastDominatingWriteInst,
1278 Acc->getRemoteInst())) {
1279 LeastDominatingWriteInst = Acc->getRemoteInst();
1280 }
1281 }
1282
1283 // Helper to determine if we can skip a specific write access.
1284 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1285 if (SkipCB && SkipCB(Acc))
1286 return true;
1287 if (!CanIgnoreThreading(Acc))
1288 return false;
1289
1290 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1291 // If we successfully excluded all effects we are interested in, the
1292 // access can be skipped.
1293 bool ReadChecked = !FindInterferingReads;
1294 bool WriteChecked = !FindInterferingWrites;
1295
1296 // If the instruction cannot reach the access, the former does not
1297 // interfere with what the access reads.
1298 if (!ReadChecked) {
1299 if (!AA::isPotentiallyReachable(A, I, *Acc.getRemoteInst(), QueryingAA,
1300 &ExclusionSet, IsLiveInCalleeCB))
1301 ReadChecked = true;
1302 }
1303 // If the instruction cannot be reach from the access, the latter does not
1304 // interfere with what the instruction reads.
1305 if (!WriteChecked) {
1306 if (!AA::isPotentiallyReachable(A, *Acc.getRemoteInst(), I, QueryingAA,
1307 &ExclusionSet, IsLiveInCalleeCB))
1308 WriteChecked = true;
1309 }
1310
1311 // If we still might be affected by the write of the access but there are
1312 // dominating writes in the function of the instruction
1313 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1314 // by them. This would have happend above if they are all in the same
1315 // function, so we only check the inter-procedural case. Effectively, we
1316 // want to show that there is no call after the dominting write that might
1317 // reach the access, and when it returns reach the instruction with the
1318 // updated value. To this end, we iterate all call sites, check if they
1319 // might reach the instruction without going through another access
1320 // (ExclusionSet) and at the same time might reach the access. However,
1321 // that is all part of AAInterFnReachability.
1322 if (!WriteChecked && HasBeenWrittenTo &&
1323 Acc.getRemoteInst()->getFunction() != &Scope) {
1324
1325 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1326 QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL);
1327
1328 // Without going backwards in the call tree, can we reach the access
1329 // from the least dominating write. Do not allow to pass the instruction
1330 // itself either.
1331 bool Inserted = ExclusionSet.insert(&I).second;
1332
1333 if (!FnReachabilityAA ||
1334 !FnReachabilityAA->instructionCanReach(
1335 A, *LeastDominatingWriteInst,
1336 *Acc.getRemoteInst()->getFunction(), &ExclusionSet))
1337 WriteChecked = true;
1338
1339 if (Inserted)
1340 ExclusionSet.erase(&I);
1341 }
1342
1343 if (ReadChecked && WriteChecked)
1344 return true;
1345
1346 if (!DT || !UseDominanceReasoning)
1347 return false;
1348 if (!DominatingWrites.count(&Acc))
1349 return false;
1350 return LeastDominatingWriteInst != Acc.getRemoteInst();
1351 };
1352
1353 // Run the user callback on all accesses we cannot skip and return if
1354 // that succeeded for all or not.
1355 for (auto &It : InterferingAccesses) {
1356 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1357 !CanSkipAccess(*It.first, It.second)) {
1358 if (!UserCB(*It.first, It.second))
1359 return false;
1360 }
1361 }
1362 return true;
1363 }
1364
1365 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1366 const AAPointerInfo &OtherAA,
1367 CallBase &CB) {
1368 using namespace AA::PointerInfo;
1369 if (!OtherAA.getState().isValidState() || !isValidState())
1370 return indicatePessimisticFixpoint();
1371
1372 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1373 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1374
1375 // Combine the accesses bin by bin.
1376 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1377 const auto &State = OtherAAImpl.getState();
1378 for (const auto &It : State) {
1379 for (auto Index : It.getSecond()) {
1380 const auto &RAcc = State.getAccess(Index);
1381 if (IsByval && !RAcc.isRead())
1382 continue;
1383 bool UsedAssumedInformation = false;
1384 AccessKind AK = RAcc.getKind();
1385 auto Content = A.translateArgumentToCallSiteContent(
1386 RAcc.getContent(), CB, *this, UsedAssumedInformation);
1387 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1388 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1389
1390 Changed |= addAccess(A, RAcc.getRanges(), CB, Content, AK,
1391 RAcc.getType(), RAcc.getRemoteInst());
1392 }
1393 }
1394 return Changed;
1395 }
1396
1397 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1398 const OffsetInfo &Offsets, CallBase &CB) {
1399 using namespace AA::PointerInfo;
1400 if (!OtherAA.getState().isValidState() || !isValidState())
1401 return indicatePessimisticFixpoint();
1402
1403 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1404
1405 // Combine the accesses bin by bin.
1406 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1407 const auto &State = OtherAAImpl.getState();
1408 for (const auto &It : State) {
1409 for (auto Index : It.getSecond()) {
1410 const auto &RAcc = State.getAccess(Index);
1411 for (auto Offset : Offsets) {
1412 auto NewRanges = Offset == AA::RangeTy::Unknown
1414 : RAcc.getRanges();
1415 if (!NewRanges.isUnknown()) {
1416 NewRanges.addToAllOffsets(Offset);
1417 }
1418 Changed |=
1419 addAccess(A, NewRanges, CB, RAcc.getContent(), RAcc.getKind(),
1420 RAcc.getType(), RAcc.getRemoteInst());
1421 }
1422 }
1423 }
1424 return Changed;
1425 }
1426
1427 /// Statistic tracking for all AAPointerInfo implementations.
1428 /// See AbstractAttribute::trackStatistics().
1429 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1430
1431 /// Dump the state into \p O.
1432 void dumpState(raw_ostream &O) {
1433 for (auto &It : OffsetBins) {
1434 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1435 << "] : " << It.getSecond().size() << "\n";
1436 for (auto AccIndex : It.getSecond()) {
1437 auto &Acc = AccessList[AccIndex];
1438 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1439 if (Acc.getLocalInst() != Acc.getRemoteInst())
1440 O << " --> " << *Acc.getRemoteInst()
1441 << "\n";
1442 if (!Acc.isWrittenValueYetUndetermined()) {
1443 if (isa_and_nonnull<Function>(Acc.getWrittenValue()))
1444 O << " - c: func " << Acc.getWrittenValue()->getName()
1445 << "\n";
1446 else if (Acc.getWrittenValue())
1447 O << " - c: " << *Acc.getWrittenValue() << "\n";
1448 else
1449 O << " - c: <unknown>\n";
1450 }
1451 }
1452 }
1453 }
1454};
1455
1456struct AAPointerInfoFloating : public AAPointerInfoImpl {
1458 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1459 : AAPointerInfoImpl(IRP, A) {}
1460
1461 /// Deal with an access and signal if it was handled successfully.
1462 bool handleAccess(Attributor &A, Instruction &I,
1463 std::optional<Value *> Content, AccessKind Kind,
1464 SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed,
1465 Type &Ty) {
1466 using namespace AA::PointerInfo;
1468 const DataLayout &DL = A.getDataLayout();
1469 TypeSize AccessSize = DL.getTypeStoreSize(&Ty);
1470 if (!AccessSize.isScalable())
1471 Size = AccessSize.getFixedValue();
1472
1473 // Make a strictly ascending list of offsets as required by addAccess()
1474 llvm::sort(Offsets);
1475 auto *Last = std::unique(Offsets.begin(), Offsets.end());
1476 Offsets.erase(Last, Offsets.end());
1477
1478 VectorType *VT = dyn_cast<VectorType>(&Ty);
1479 if (!VT || VT->getElementCount().isScalable() ||
1480 !Content.value_or(nullptr) || !isa<Constant>(*Content) ||
1481 (*Content)->getType() != VT ||
1482 DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
1483 Changed = Changed | addAccess(A, {Offsets, Size}, I, Content, Kind, &Ty);
1484 } else {
1485 // Handle vector stores with constant content element-wise.
1486 // TODO: We could look for the elements or create instructions
1487 // representing them.
1488 // TODO: We need to push the Content into the range abstraction
1489 // (AA::RangeTy) to allow different content values for different
1490 // ranges. ranges. Hence, support vectors storing different values.
1491 Type *ElementType = VT->getElementType();
1492 int64_t ElementSize = DL.getTypeStoreSize(ElementType).getFixedValue();
1493 auto *ConstContent = cast<Constant>(*Content);
1494 Type *Int32Ty = Type::getInt32Ty(ElementType->getContext());
1495 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1496
1497 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1498 Value *ElementContent = ConstantExpr::getExtractElement(
1499 ConstContent, ConstantInt::get(Int32Ty, i));
1500
1501 // Add the element access.
1502 Changed = Changed | addAccess(A, {ElementOffsets, ElementSize}, I,
1503 ElementContent, Kind, ElementType);
1504
1505 // Advance the offsets for the next element.
1506 for (auto &ElementOffset : ElementOffsets)
1507 ElementOffset += ElementSize;
1508 }
1509 }
1510 return true;
1511 };
1512
1513 /// See AbstractAttribute::updateImpl(...).
1514 ChangeStatus updateImpl(Attributor &A) override;
1515
1516 /// If the indices to \p GEP can be traced to constants, incorporate all
1517 /// of these into \p UsrOI.
1518 ///
1519 /// \return true iff \p UsrOI is updated.
1520 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1521 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1522 const GEPOperator *GEP);
1523
1524 /// See AbstractAttribute::trackStatistics()
1525 void trackStatistics() const override {
1526 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1527 }
1528};
1529
1530bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1531 const DataLayout &DL,
1532 OffsetInfo &UsrOI,
1533 const OffsetInfo &PtrOI,
1534 const GEPOperator *GEP) {
1535 unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
1536 MapVector<Value *, APInt> VariableOffsets;
1537 APInt ConstantOffset(BitWidth, 0);
1538
1539 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1540 "Don't look for constant values if the offset has already been "
1541 "determined to be unknown.");
1542
1543 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1544 UsrOI.setUnknown();
1545 return true;
1546 }
1547
1548 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1549 << (VariableOffsets.empty() ? "" : "not") << " constant "
1550 << *GEP << "\n");
1551
1552 auto Union = PtrOI;
1553 Union.addToAll(ConstantOffset.getSExtValue());
1554
1555 // Each VI in VariableOffsets has a set of potential constant values. Every
1556 // combination of elements, picked one each from these sets, is separately
1557 // added to the original set of offsets, thus resulting in more offsets.
1558 for (const auto &VI : VariableOffsets) {
1559 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1560 *this, IRPosition::value(*VI.first), DepClassTy::OPTIONAL);
1561 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1562 UsrOI.setUnknown();
1563 return true;
1564 }
1565
1566 // UndefValue is treated as a zero, which leaves Union as is.
1567 if (PotentialConstantsAA->undefIsContained())
1568 continue;
1569
1570 // We need at least one constant in every set to compute an actual offset.
1571 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1572 // don't actually exist. In other words, the absence of constant values
1573 // implies that the operation can be assumed dead for now.
1574 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1575 if (AssumedSet.empty())
1576 return false;
1577
1578 OffsetInfo Product;
1579 for (const auto &ConstOffset : AssumedSet) {
1580 auto CopyPerOffset = Union;
1581 CopyPerOffset.addToAll(ConstOffset.getSExtValue() *
1582 VI.second.getZExtValue());
1583 Product.merge(CopyPerOffset);
1584 }
1585 Union = Product;
1586 }
1587
1588 UsrOI = std::move(Union);
1589 return true;
1590}
1591
1592ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1593 using namespace AA::PointerInfo;
1595 const DataLayout &DL = A.getDataLayout();
1596 Value &AssociatedValue = getAssociatedValue();
1597
1598 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1599 OffsetInfoMap[&AssociatedValue].insert(0);
1600
1601 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1602 // One does not simply walk into a map and assign a reference to a possibly
1603 // new location. That can cause an invalidation before the assignment
1604 // happens, like so:
1605 //
1606 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1607 //
1608 // The RHS is a reference that may be invalidated by an insertion caused by
1609 // the LHS. So we ensure that the side-effect of the LHS happens first.
1610 auto &UsrOI = OffsetInfoMap[Usr];
1611 auto &PtrOI = OffsetInfoMap[CurPtr];
1612 assert(!PtrOI.isUnassigned() &&
1613 "Cannot pass through if the input Ptr was not visited!");
1614 UsrOI = PtrOI;
1615 Follow = true;
1616 return true;
1617 };
1618
1619 const auto *F = getAnchorScope();
1620 const auto *CI =
1621 F ? A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(*F)
1622 : nullptr;
1623 const auto *TLI =
1624 F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
1625
1626 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1627 Value *CurPtr = U.get();
1628 User *Usr = U.getUser();
1629 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1630 << "\n");
1631 assert(OffsetInfoMap.count(CurPtr) &&
1632 "The current pointer offset should have been seeded!");
1633
1634 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
1635 if (CE->isCast())
1636 return HandlePassthroughUser(Usr, CurPtr, Follow);
1637 if (CE->isCompare())
1638 return true;
1639 if (!isa<GEPOperator>(CE)) {
1640 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1641 << "\n");
1642 return false;
1643 }
1644 }
1645 if (auto *GEP = dyn_cast<GEPOperator>(Usr)) {
1646 // Note the order here, the Usr access might change the map, CurPtr is
1647 // already in it though.
1648 auto &UsrOI = OffsetInfoMap[Usr];
1649 auto &PtrOI = OffsetInfoMap[CurPtr];
1650
1651 if (UsrOI.isUnknown())
1652 return true;
1653
1654 if (PtrOI.isUnknown()) {
1655 Follow = true;
1656 UsrOI.setUnknown();
1657 return true;
1658 }
1659
1660 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1661 return true;
1662 }
1663 if (isa<PtrToIntInst>(Usr))
1664 return false;
1665 if (isa<CastInst>(Usr) || isa<SelectInst>(Usr) || isa<ReturnInst>(Usr))
1666 return HandlePassthroughUser(Usr, CurPtr, Follow);
1667
1668 // For PHIs we need to take care of the recurrence explicitly as the value
1669 // might change while we iterate through a loop. For now, we give up if
1670 // the PHI is not invariant.
1671 if (isa<PHINode>(Usr)) {
1672 // Note the order here, the Usr access might change the map, CurPtr is
1673 // already in it though.
1674 bool IsFirstPHIUser = !OffsetInfoMap.count(Usr);
1675 auto &UsrOI = OffsetInfoMap[Usr];
1676 auto &PtrOI = OffsetInfoMap[CurPtr];
1677
1678 // Check if the PHI operand has already an unknown offset as we can't
1679 // improve on that anymore.
1680 if (PtrOI.isUnknown()) {
1681 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1682 << *CurPtr << " in " << *Usr << "\n");
1683 Follow = !UsrOI.isUnknown();
1684 UsrOI.setUnknown();
1685 return true;
1686 }
1687
1688 // Check if the PHI is invariant (so far).
1689 if (UsrOI == PtrOI) {
1690 assert(!PtrOI.isUnassigned() &&
1691 "Cannot assign if the current Ptr was not visited!");
1692 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1693 return true;
1694 }
1695
1696 // Check if the PHI operand can be traced back to AssociatedValue.
1697 APInt Offset(
1698 DL.getIndexSizeInBits(CurPtr->getType()->getPointerAddressSpace()),
1699 0);
1700 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1701 DL, Offset, /* AllowNonInbounds */ true);
1702 auto It = OffsetInfoMap.find(CurPtrBase);
1703 if (It == OffsetInfoMap.end()) {
1704 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1705 << *CurPtr << " in " << *Usr << "\n");
1706 UsrOI.setUnknown();
1707 Follow = true;
1708 return true;
1709 }
1710
1711 // Check if the PHI operand is not dependent on the PHI itself. Every
1712 // recurrence is a cyclic net of PHIs in the data flow, and has an
1713 // equivalent Cycle in the control flow. One of those PHIs must be in the
1714 // header of that control flow Cycle. This is independent of the choice of
1715 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1716 // every Cycle header; if such a node is marked unknown, this will
1717 // eventually propagate through the whole net of PHIs in the recurrence.
1718 if (mayBeInCycle(CI, cast<Instruction>(Usr), /* HeaderOnly */ true)) {
1719 auto BaseOI = It->getSecond();
1720 BaseOI.addToAll(Offset.getZExtValue());
1721 if (IsFirstPHIUser || BaseOI == UsrOI) {
1722 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1723 << " in " << *Usr << "\n");
1724 return HandlePassthroughUser(Usr, CurPtr, Follow);
1725 }
1726
1727 LLVM_DEBUG(
1728 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1729 << *CurPtr << " in " << *Usr << "\n");
1730 UsrOI.setUnknown();
1731 Follow = true;
1732 return true;
1733 }
1734
1735 UsrOI.merge(PtrOI);
1736 Follow = true;
1737 return true;
1738 }
1739
1740 if (auto *LoadI = dyn_cast<LoadInst>(Usr)) {
1741 // If the access is to a pointer that may or may not be the associated
1742 // value, e.g. due to a PHI, we cannot assume it will be read.
1743 AccessKind AK = AccessKind::AK_R;
1744 if (getUnderlyingObject(CurPtr) == &AssociatedValue)
1745 AK = AccessKind(AK | AccessKind::AK_MUST);
1746 else
1747 AK = AccessKind(AK | AccessKind::AK_MAY);
1748 if (!handleAccess(A, *LoadI, /* Content */ nullptr, AK,
1749 OffsetInfoMap[CurPtr].Offsets, Changed,
1750 *LoadI->getType()))
1751 return false;
1752
1753 auto IsAssumption = [](Instruction &I) {
1754 if (auto *II = dyn_cast<IntrinsicInst>(&I))
1755 return II->isAssumeLikeIntrinsic();
1756 return false;
1757 };
1758
1759 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1760 // Check if the assumption and the load are executed together without
1761 // memory modification.
1762 do {
1763 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1764 return true;
1765 FromI = FromI->getNextNonDebugInstruction();
1766 } while (FromI && FromI != ToI);
1767 return false;
1768 };
1769
1770 BasicBlock *BB = LoadI->getParent();
1771 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1772 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1773 return false;
1774 BasicBlock *IntrBB = IntrI.getParent();
1775 if (IntrI.getParent() == BB) {
1776 if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), &IntrI))
1777 return false;
1778 } else {
1779 auto PredIt = pred_begin(IntrBB);
1780 if (PredIt == pred_end(IntrBB))
1781 return false;
1782 if ((*PredIt) != BB)
1783 return false;
1784 if (++PredIt != pred_end(IntrBB))
1785 return false;
1786 for (auto *SuccBB : successors(BB)) {
1787 if (SuccBB == IntrBB)
1788 continue;
1789 if (isa<UnreachableInst>(SuccBB->getTerminator()))
1790 continue;
1791 return false;
1792 }
1793 if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(),
1794 BB->getTerminator()))
1795 return false;
1796 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1797 return false;
1798 }
1799 return true;
1800 };
1801
1802 std::pair<Value *, IntrinsicInst *> Assumption;
1803 for (const Use &LoadU : LoadI->uses()) {
1804 if (auto *CmpI = dyn_cast<CmpInst>(LoadU.getUser())) {
1805 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1806 continue;
1807 for (const Use &CmpU : CmpI->uses()) {
1808 if (auto *IntrI = dyn_cast<IntrinsicInst>(CmpU.getUser())) {
1809 if (!IsValidAssume(*IntrI))
1810 continue;
1811 int Idx = CmpI->getOperandUse(0) == LoadU;
1812 Assumption = {CmpI->getOperand(Idx), IntrI};
1813 break;
1814 }
1815 }
1816 }
1817 if (Assumption.first)
1818 break;
1819 }
1820
1821 // Check if we found an assumption associated with this load.
1822 if (!Assumption.first || !Assumption.second)
1823 return true;
1824
1825 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1826 << *Assumption.second << ": " << *LoadI
1827 << " == " << *Assumption.first << "\n");
1828 bool UsedAssumedInformation = false;
1829 std::optional<Value *> Content = nullptr;
1830 if (Assumption.first)
1831 Content =
1832 A.getAssumedSimplified(*Assumption.first, *this,
1833 UsedAssumedInformation, AA::Interprocedural);
1834 return handleAccess(
1835 A, *Assumption.second, Content, AccessKind::AK_ASSUMPTION,
1836 OffsetInfoMap[CurPtr].Offsets, Changed, *LoadI->getType());
1837 }
1838
1839 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1840 ArrayRef<Value *> OtherOps, AccessKind AK) {
1841 for (auto *OtherOp : OtherOps) {
1842 if (OtherOp == CurPtr) {
1843 LLVM_DEBUG(
1844 dbgs()
1845 << "[AAPointerInfo] Escaping use in store like instruction " << I
1846 << "\n");
1847 return false;
1848 }
1849 }
1850
1851 // If the access is to a pointer that may or may not be the associated
1852 // value, e.g. due to a PHI, we cannot assume it will be written.
1853 if (getUnderlyingObject(CurPtr) == &AssociatedValue)
1854 AK = AccessKind(AK | AccessKind::AK_MUST);
1855 else
1856 AK = AccessKind(AK | AccessKind::AK_MAY);
1857 bool UsedAssumedInformation = false;
1858 std::optional<Value *> Content = nullptr;
1859 if (ValueOp)
1860 Content = A.getAssumedSimplified(
1861 *ValueOp, *this, UsedAssumedInformation, AA::Interprocedural);
1862 return handleAccess(A, I, Content, AK, OffsetInfoMap[CurPtr].Offsets,
1863 Changed, ValueTy);
1864 };
1865
1866 if (auto *StoreI = dyn_cast<StoreInst>(Usr))
1867 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1868 *StoreI->getValueOperand()->getType(),
1869 {StoreI->getValueOperand()}, AccessKind::AK_W);
1870 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Usr))
1871 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1872 {RMWI->getValOperand()}, AccessKind::AK_RW);
1873 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Usr))
1874 return HandleStoreLike(
1875 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1876 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1877 AccessKind::AK_RW);
1878
1879 if (auto *CB = dyn_cast<CallBase>(Usr)) {
1880 if (CB->isLifetimeStartOrEnd())
1881 return true;
1882 if (getFreedOperand(CB, TLI) == U)
1883 return true;
1884 if (CB->isArgOperand(&U)) {
1885 unsigned ArgNo = CB->getArgOperandNo(&U);
1886 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1887 *this, IRPosition::callsite_argument(*CB, ArgNo),
1889 if (!CSArgPI)
1890 return false;
1891 Changed =
1892 translateAndAddState(A, *CSArgPI, OffsetInfoMap[CurPtr], *CB) |
1893 Changed;
1894 return isValidState();
1895 }
1896 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1897 << "\n");
1898 // TODO: Allow some call uses
1899 return false;
1900 }
1901
1902 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1903 return false;
1904 };
1905 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1906 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1907 if (OffsetInfoMap.count(NewU)) {
1908 LLVM_DEBUG({
1909 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1910 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1911 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1912 << "\n";
1913 }
1914 });
1915 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1916 }
1917 OffsetInfoMap[NewU] = OffsetInfoMap[OldU];
1918 return true;
1919 };
1920 if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
1921 /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
1922 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1923 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1924 return indicatePessimisticFixpoint();
1925 }
1926
1927 LLVM_DEBUG({
1928 dbgs() << "Accesses by bin after update:\n";
1929 dumpState(dbgs());
1930 });
1931
1932 return Changed;
1933}
1934
1935struct AAPointerInfoReturned final : AAPointerInfoImpl {
1936 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1937 : AAPointerInfoImpl(IRP, A) {}
1938
1939 /// See AbstractAttribute::updateImpl(...).
1940 ChangeStatus updateImpl(Attributor &A) override {
1941 return indicatePessimisticFixpoint();
1942 }
1943
1944 /// See AbstractAttribute::trackStatistics()
1945 void trackStatistics() const override {
1946 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1947 }
1948};
1949
1950struct AAPointerInfoArgument final : AAPointerInfoFloating {
1951 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1952 : AAPointerInfoFloating(IRP, A) {}
1953
1954 /// See AbstractAttribute::trackStatistics()
1955 void trackStatistics() const override {
1956 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1957 }
1958};
1959
1960struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1961 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1962 : AAPointerInfoFloating(IRP, A) {}
1963
1964 /// See AbstractAttribute::updateImpl(...).
1965 ChangeStatus updateImpl(Attributor &A) override {
1966 using namespace AA::PointerInfo;
1967 // We handle memory intrinsics explicitly, at least the first (=
1968 // destination) and second (=source) arguments as we know how they are
1969 // accessed.
1970 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {
1971 ConstantInt *Length = dyn_cast<ConstantInt>(MI->getLength());
1972 int64_t LengthVal = AA::RangeTy::Unknown;
1973 if (Length)
1974 LengthVal = Length->getSExtValue();
1975 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1976 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1977 if (ArgNo > 1) {
1978 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1979 << *MI << "\n");
1980 return indicatePessimisticFixpoint();
1981 } else {
1982 auto Kind =
1983 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1984 Changed =
1985 Changed | addAccess(A, {0, LengthVal}, *MI, nullptr, Kind, nullptr);
1986 }
1987 LLVM_DEBUG({
1988 dbgs() << "Accesses by bin after update:\n";
1989 dumpState(dbgs());
1990 });
1991
1992 return Changed;
1993 }
1994
1995 // TODO: Once we have call site specific value information we can provide
1996 // call site specific liveness information and then it makes
1997 // sense to specialize attributes for call sites arguments instead of
1998 // redirecting requests to the callee argument.
1999 Argument *Arg = getAssociatedArgument();
2000 if (Arg) {
2001 const IRPosition &ArgPos = IRPosition::argument(*Arg);
2002 auto *ArgAA =
2003 A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED);
2004 if (ArgAA && ArgAA->getState().isValidState())
2005 return translateAndAddStateFromCallee(A, *ArgAA,
2006 *cast<CallBase>(getCtxI()));
2007 if (!Arg->getParent()->isDeclaration())
2008 return indicatePessimisticFixpoint();
2009 }
2010
2011 bool IsKnownNoCapture;
2012 if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
2013 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
2014 return indicatePessimisticFixpoint();
2015
2016 bool IsKnown = false;
2017 if (AA::isAssumedReadNone(A, getIRPosition(), *this, IsKnown))
2018 return ChangeStatus::UNCHANGED;
2019 bool ReadOnly = AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown);
2020 auto Kind =
2021 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2022 return addAccess(A, AA::RangeTy::getUnknown(), *getCtxI(), nullptr, Kind,
2023 nullptr);
2024 }
2025
2026 /// See AbstractAttribute::trackStatistics()
2027 void trackStatistics() const override {
2028 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2029 }
2030};
2031
2032struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2033 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2034 : AAPointerInfoFloating(IRP, A) {}
2035
2036 /// See AbstractAttribute::trackStatistics()
2037 void trackStatistics() const override {
2038 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2039 }
2040};
2041} // namespace
2042
2043/// -----------------------NoUnwind Function Attribute--------------------------
2044
2045namespace {
2046struct AANoUnwindImpl : AANoUnwind {
2047 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2048
2049 /// See AbstractAttribute::initialize(...).
2050 void initialize(Attributor &A) override {
2051 bool IsKnown;
2052 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2053 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2054 (void)IsKnown;
2055 }
2056
2057 const std::string getAsStr(Attributor *A) const override {
2058 return getAssumed() ? "nounwind" : "may-unwind";
2059 }
2060
2061 /// See AbstractAttribute::updateImpl(...).
2062 ChangeStatus updateImpl(Attributor &A) override {
2063 auto Opcodes = {
2064 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2065 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2066 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2067
2068 auto CheckForNoUnwind = [&](Instruction &I) {
2069 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2070 return true;
2071
2072 if (const auto *CB = dyn_cast<CallBase>(&I)) {
2073 bool IsKnownNoUnwind;
2074 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2075 A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
2076 IsKnownNoUnwind);
2077 }
2078 return false;
2079 };
2080
2081 bool UsedAssumedInformation = false;
2082 if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,
2083 UsedAssumedInformation))
2084 return indicatePessimisticFixpoint();
2085
2086 return ChangeStatus::UNCHANGED;
2087 }
2088};
2089
2090struct AANoUnwindFunction final : public AANoUnwindImpl {
2091 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2092 : AANoUnwindImpl(IRP, A) {}
2093
2094 /// See AbstractAttribute::trackStatistics()
2095 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2096};
2097
2098/// NoUnwind attribute deduction for a call sites.
2099struct AANoUnwindCallSite final
2100 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2101 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2102 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2103
2104 /// See AbstractAttribute::trackStatistics()
2105 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2106};
2107} // namespace
2108
2109/// ------------------------ NoSync Function Attribute -------------------------
2110
2111bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2112 switch (CB.getIntrinsicID()) {
2113 case Intrinsic::nvvm_barrier0:
2114 case Intrinsic::nvvm_barrier0_and:
2115 case Intrinsic::nvvm_barrier0_or:
2116 case Intrinsic::nvvm_barrier0_popc:
2117 return true;
2118 case Intrinsic::amdgcn_s_barrier:
2119 if (ExecutedAligned)
2120 return true;
2121 break;
2122 default:
2123 break;
2124 }
2125 return hasAssumption(CB, KnownAssumptionString("ompx_aligned_barrier"));
2126}
2127
2129 if (!I->isAtomic())
2130 return false;
2131
2132 if (auto *FI = dyn_cast<FenceInst>(I))
2133 // All legal orderings for fence are stronger than monotonic.
2134 return FI->getSyncScopeID() != SyncScope::SingleThread;
2135 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
2136 // Unordered is not a legal ordering for cmpxchg.
2137 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2138 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2139 }
2140
2141 AtomicOrdering Ordering;
2142 switch (I->getOpcode()) {
2143 case Instruction::AtomicRMW:
2144 Ordering = cast<AtomicRMWInst>(I)->getOrdering();
2145 break;
2146 case Instruction::Store:
2147 Ordering = cast<StoreInst>(I)->getOrdering();
2148 break;
2149 case Instruction::Load:
2150 Ordering = cast<LoadInst>(I)->getOrdering();
2151 break;
2152 default:
2154 "New atomic operations need to be known in the attributor.");
2155 }
2156
2157 return (Ordering != AtomicOrdering::Unordered &&
2158 Ordering != AtomicOrdering::Monotonic);
2159}
2160
2161/// Return true if this intrinsic is nosync. This is only used for intrinsics
2162/// which would be nosync except that they have a volatile flag. All other
2163/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
2165 if (auto *MI = dyn_cast<MemIntrinsic>(I))
2166 return !MI->isVolatile();
2167 return false;
2168}
2169
2170namespace {
2171struct AANoSyncImpl : AANoSync {
2172 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2173
2174 /// See AbstractAttribute::initialize(...).
2175 void initialize(Attributor &A) override {
2176 bool IsKnown;
2177 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2178 DepClassTy::NONE, IsKnown));
2179 (void)IsKnown;
2180 }
2181
2182 const std::string getAsStr(Attributor *A) const override {
2183 return getAssumed() ? "nosync" : "may-sync";
2184 }
2185
2186 /// See AbstractAttribute::updateImpl(...).
2187 ChangeStatus updateImpl(Attributor &A) override;
2188};
2189
2190ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2191
2192 auto CheckRWInstForNoSync = [&](Instruction &I) {
2193 return AA::isNoSyncInst(A, I, *this);
2194 };
2195
2196 auto CheckForNoSync = [&](Instruction &I) {
2197 // At this point we handled all read/write effects and they are all
2198 // nosync, so they can be skipped.
2199 if (I.mayReadOrWriteMemory())
2200 return true;
2201
2202 bool IsKnown;
2203 CallBase &CB = cast<CallBase>(I);
2204 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2206 IsKnown))
2207 return true;
2208
2209 // non-convergent and readnone imply nosync.
2210 return !CB.isConvergent();
2211 };
2212
2213 bool UsedAssumedInformation = false;
2214 if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,
2215 UsedAssumedInformation) ||
2216 !A.checkForAllCallLikeInstructions(CheckForNoSync, *this,
2217 UsedAssumedInformation))
2218 return indicatePessimisticFixpoint();
2219
2221}
2222
2223struct AANoSyncFunction final : public AANoSyncImpl {
2224 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2225 : AANoSyncImpl(IRP, A) {}
2226
2227 /// See AbstractAttribute::trackStatistics()
2228 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2229};
2230
2231/// NoSync attribute deduction for a call sites.
2232struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2233 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2234 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2235
2236 /// See AbstractAttribute::trackStatistics()
2237 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2238};
2239} // namespace
2240
2241/// ------------------------ No-Free Attributes ----------------------------
2242
2243namespace {
2244struct AANoFreeImpl : public AANoFree {
2245 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2246
2247 /// See AbstractAttribute::initialize(...).
2248 void initialize(Attributor &A) override {
2249 bool IsKnown;
2250 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2251 DepClassTy::NONE, IsKnown));
2252 (void)IsKnown;
2253 }
2254
2255 /// See AbstractAttribute::updateImpl(...).
2256 ChangeStatus updateImpl(Attributor &A) override {
2257 auto CheckForNoFree = [&](Instruction &I) {
2258 bool IsKnown;
2259 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2260 A, this, IRPosition::callsite_function(cast<CallBase>(I)),
2261 DepClassTy::REQUIRED, IsKnown);
2262 };
2263
2264 bool UsedAssumedInformation = false;
2265 if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this,
2266 UsedAssumedInformation))
2267 return indicatePessimisticFixpoint();
2268 return ChangeStatus::UNCHANGED;
2269 }
2270
2271 /// See AbstractAttribute::getAsStr().
2272 const std::string getAsStr(Attributor *A) const override {
2273 return getAssumed() ? "nofree" : "may-free";
2274 }
2275};
2276
2277struct AANoFreeFunction final : public AANoFreeImpl {
2278 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2279 : AANoFreeImpl(IRP, A) {}
2280
2281 /// See AbstractAttribute::trackStatistics()
2282 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2283};
2284
2285/// NoFree attribute deduction for a call sites.
2286struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2287 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2288 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2289
2290 /// See AbstractAttribute::trackStatistics()
2291 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2292};
2293
2294/// NoFree attribute for floating values.
2295struct AANoFreeFloating : AANoFreeImpl {
2296 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2297 : AANoFreeImpl(IRP, A) {}
2298
2299 /// See AbstractAttribute::trackStatistics()
2300 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2301
2302 /// See Abstract Attribute::updateImpl(...).
2303 ChangeStatus updateImpl(Attributor &A) override {
2304 const IRPosition &IRP = getIRPosition();
2305
2306 bool IsKnown;
2307 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this,
2309 DepClassTy::OPTIONAL, IsKnown))
2310 return ChangeStatus::UNCHANGED;
2311
2312 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2313 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2314 Instruction *UserI = cast<Instruction>(U.getUser());
2315 if (auto *CB = dyn_cast<CallBase>(UserI)) {
2316 if (CB->isBundleOperand(&U))
2317 return false;
2318 if (!CB->isArgOperand(&U))
2319 return true;
2320 unsigned ArgNo = CB->getArgOperandNo(&U);
2321
2322 bool IsKnown;
2323 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2324 A, this, IRPosition::callsite_argument(*CB, ArgNo),
2325 DepClassTy::REQUIRED, IsKnown);
2326 }
2327
2328 if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
2329 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
2330 Follow = true;
2331 return true;
2332 }
2333 if (isa<StoreInst>(UserI) || isa<LoadInst>(UserI) ||
2334 isa<ReturnInst>(UserI))
2335 return true;
2336
2337 // Unknown user.
2338 return false;
2339 };
2340 if (!A.checkForAllUses(Pred, *this, AssociatedValue))
2341 return indicatePessimisticFixpoint();
2342
2343 return ChangeStatus::UNCHANGED;
2344 }
2345};
2346
2347/// NoFree attribute for a call site argument.
2348struct AANoFreeArgument final : AANoFreeFloating {
2349 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2350 : AANoFreeFloating(IRP, A) {}
2351
2352 /// See AbstractAttribute::trackStatistics()
2353 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2354};
2355
2356/// NoFree attribute for call site arguments.
2357struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2358 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2359 : AANoFreeFloating(IRP, A) {}
2360
2361 /// See AbstractAttribute::updateImpl(...).
2362 ChangeStatus updateImpl(Attributor &A) override {
2363 // TODO: Once we have call site specific value information we can provide
2364 // call site specific liveness information and then it makes
2365 // sense to specialize attributes for call sites arguments instead of
2366 // redirecting requests to the callee argument.
2367 Argument *Arg = getAssociatedArgument();
2368 if (!Arg)
2369 return indicatePessimisticFixpoint();
2370 const IRPosition &ArgPos = IRPosition::argument(*Arg);
2371 bool IsKnown;
2372 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos,
2373 DepClassTy::REQUIRED, IsKnown))
2374 return ChangeStatus::UNCHANGED;
2375 return indicatePessimisticFixpoint();
2376 }
2377
2378 /// See AbstractAttribute::trackStatistics()
2379 void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nofree)};
2380};
2381
2382/// NoFree attribute for function return value.
2383struct AANoFreeReturned final : AANoFreeFloating {
2384 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2385 : AANoFreeFloating(IRP, A) {
2386 llvm_unreachable("NoFree is not applicable to function returns!");
2387 }
2388
2389 /// See AbstractAttribute::initialize(...).
2390 void initialize(Attributor &A) override {
2391 llvm_unreachable("NoFree is not applicable to function returns!");
2392 }
2393
2394 /// See AbstractAttribute::updateImpl(...).
2395 ChangeStatus updateImpl(Attributor &A) override {
2396 llvm_unreachable("NoFree is not applicable to function returns!");
2397 }
2398
2399 /// See AbstractAttribute::trackStatistics()
2400 void trackStatistics() const override {}
2401};
2402
2403/// NoFree attribute deduction for a call site return value.
2404struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2405 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2406 : AANoFreeFloating(IRP, A) {}
2407
2408 ChangeStatus manifest(Attributor &A) override {
2409 return ChangeStatus::UNCHANGED;
2410 }
2411 /// See AbstractAttribute::trackStatistics()
2412 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2413};
2414} // namespace
2415
2416/// ------------------------ NonNull Argument Attribute ------------------------
2417
2419 Attribute::AttrKind ImpliedAttributeKind,
2420 bool IgnoreSubsumingPositions) {
2422 AttrKinds.push_back(Attribute::NonNull);
2425 AttrKinds.push_back(Attribute::Dereferenceable);
2426 if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
2427 return true;
2428
2429 DominatorTree *DT = nullptr;
2430 AssumptionCache *AC = nullptr;
2431 InformationCache &InfoCache = A.getInfoCache();
2432 if (const Function *Fn = IRP.getAnchorScope()) {
2433 if (!Fn->isDeclaration()) {
2436 }
2437 }
2438
2440 if (IRP.getPositionKind() != IRP_RETURNED) {
2441 Worklist.push_back({IRP.getAssociatedValue(), IRP.getCtxI()});
2442 } else {
2443 bool UsedAssumedInformation = false;
2444 if (!A.checkForAllInstructions(
2445 [&](Instruction &I) {
2446 Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});
2447 return true;
2448 },
2449 IRP.getAssociatedFunction(), nullptr, {Instruction::Ret},
2450 UsedAssumedInformation))
2451 return false;
2452 }
2453
2454 if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
2455 return !isKnownNonZero(VAC.getValue(), A.getDataLayout(), 0, AC,
2456 VAC.getCtxI(), DT);
2457 }))
2458 return false;
2459
2460 A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),
2461 Attribute::NonNull)});
2462 return true;
2463}
2464
2465namespace {
2466static int64_t getKnownNonNullAndDerefBytesForUse(
2467 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2468 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2469 TrackUse = false;
2470
2471 const Value *UseV = U->get();
2472 if (!UseV->getType()->isPointerTy())
2473 return 0;
2474
2475 // We need to follow common pointer manipulation uses to the accesses they
2476 // feed into. We can try to be smart to avoid looking through things we do not
2477 // like for now, e.g., non-inbounds GEPs.
2478 if (isa<CastInst>(I)) {
2479 TrackUse = true;
2480 return 0;
2481 }
2482
2483 if (isa<GetElementPtrInst>(I)) {
2484 TrackUse = true;
2485 return 0;
2486 }
2487
2488 Type *PtrTy = UseV->getType();
2489 const Function *F = I->getFunction();
2492 const DataLayout &DL = A.getInfoCache().getDL();
2493 if (const auto *CB = dyn_cast<CallBase>(I)) {
2494 if (CB->isBundleOperand(U)) {
2496 U, {Attribute::NonNull, Attribute::Dereferenceable})) {
2497 IsNonNull |=
2498 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2499 return RK.ArgValue;
2500 }
2501 return 0;
2502 }
2503
2504 if (CB->isCallee(U)) {
2505 IsNonNull |= !NullPointerIsDefined;
2506 return 0;
2507 }
2508
2509 unsigned ArgNo = CB->getArgOperandNo(U);
2510 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
2511 // As long as we only use known information there is no need to track
2512 // dependences here.
2513 bool IsKnownNonNull;
2514 AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP,
2515 DepClassTy::NONE, IsKnownNonNull);
2516 IsNonNull |= IsKnownNonNull;
2517 auto *DerefAA =
2518 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClassTy::NONE);
2519 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2520 }
2521
2522 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
2523 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2524 Loc->Size.isScalable() || I->isVolatile())
2525 return 0;
2526
2527 int64_t Offset;
2528 const Value *Base =
2529 getMinimalBaseOfPointer(A, QueryingAA, Loc->Ptr, Offset, DL);
2530 if (Base && Base == &AssociatedValue) {
2531 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2532 IsNonNull |= !NullPointerIsDefined;
2533 return std::max(int64_t(0), DerefBytes);
2534 }
2535
2536 /// Corner case when an offset is 0.
2538 /*AllowNonInbounds*/ true);
2539 if (Base && Base == &AssociatedValue && Offset == 0) {
2540 int64_t DerefBytes = Loc->Size.getValue();
2541 IsNonNull |= !NullPointerIsDefined;
2542 return std::max(int64_t(0), DerefBytes);
2543 }
2544
2545 return 0;
2546}
2547
2548struct AANonNullImpl : AANonNull {
2549 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2550
2551 /// See AbstractAttribute::initialize(...).
2552 void initialize(Attributor &A) override {
2553 Value &V = *getAssociatedValue().stripPointerCasts();
2554 if (isa<ConstantPointerNull>(V)) {
2555 indicatePessimisticFixpoint();
2556 return;
2557 }
2558
2559 if (Instruction *CtxI = getCtxI())
2560 followUsesInMBEC(*this, A, getState(), *CtxI);
2561 }
2562
2563 /// See followUsesInMBEC
2564 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2565 AANonNull::StateType &State) {
2566 bool IsNonNull = false;
2567 bool TrackUse = false;
2568 getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,
2569 IsNonNull, TrackUse);
2570 State.setKnown(IsNonNull);
2571 return TrackUse;
2572 }
2573
2574 /// See AbstractAttribute::getAsStr().
2575 const std::string getAsStr(Attributor *A) const override {
2576 return getAssumed() ? "nonnull" : "may-null";
2577 }
2578};
2579
2580/// NonNull attribute for a floating value.
2581struct AANonNullFloating : public AANonNullImpl {
2582 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2583 : AANonNullImpl(IRP, A) {}
2584
2585 /// See AbstractAttribute::updateImpl(...).
2586 ChangeStatus updateImpl(Attributor &A) override {
2587 auto CheckIRP = [&](const IRPosition &IRP) {
2588 bool IsKnownNonNull;
2589 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2590 A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
2591 };
2592
2593 bool Stripped;
2594 bool UsedAssumedInformation = false;
2595 Value *AssociatedValue = &getAssociatedValue();
2597 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
2598 AA::AnyScope, UsedAssumedInformation))
2599 Stripped = false;
2600 else
2601 Stripped =
2602 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2603
2604 if (!Stripped) {
2605 bool IsKnown;
2606 if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))
2607 if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {
2608 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2609 A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
2610 IsKnown);
2611 }))
2612 return ChangeStatus::UNCHANGED;
2613 if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))
2614 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2615 A, this, IRPosition::value(*Select->getFalseValue()),
2616 DepClassTy::OPTIONAL, IsKnown) &&
2617 AA::hasAssumedIRAttr<Attribute::NonNull>(
2618 A, this, IRPosition::value(*Select->getTrueValue()),
2619 DepClassTy::OPTIONAL, IsKnown))
2620 return ChangeStatus::UNCHANGED;
2621
2622 // If we haven't stripped anything we might still be able to use a
2623 // different AA, but only if the IRP changes. Effectively when we
2624 // interpret this not as a call site value but as a floating/argument
2625 // value.
2626 const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
2627 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2628 return indicatePessimisticFixpoint();
2629 return ChangeStatus::UNCHANGED;
2630 }
2631
2632 for (const auto &VAC : Values)
2633 if (!CheckIRP(IRPosition::value(*VAC.getValue())))
2634 return indicatePessimisticFixpoint();
2635
2636 return ChangeStatus::UNCHANGED;
2637 }
2638
2639 /// See AbstractAttribute::trackStatistics()
2640 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2641};
2642
2643/// NonNull attribute for function return value.
2644struct AANonNullReturned final
2645 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2646 false, AANonNull::IRAttributeKind, false> {
2647 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2648 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2649 false, Attribute::NonNull, false>(IRP, A) {
2650 }
2651
2652 /// See AbstractAttribute::getAsStr().
2653 const std::string getAsStr(Attributor *A) const override {
2654 return getAssumed() ? "nonnull" : "may-null";
2655 }
2656
2657 /// See AbstractAttribute::trackStatistics()
2658 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2659};
2660
2661/// NonNull attribute for function argument.
2662struct AANonNullArgument final
2663 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2664 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2665 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2666
2667 /// See AbstractAttribute::trackStatistics()
2668 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2669};
2670
2671struct AANonNullCallSiteArgument final : AANonNullFloating {
2672 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2673 : AANonNullFloating(IRP, A) {}
2674
2675 /// See AbstractAttribute::trackStatistics()
2676 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2677};
2678
2679/// NonNull attribute for a call site return position.
2680struct AANonNullCallSiteReturned final
2681 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2682 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2683 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2684
2685 /// See AbstractAttribute::trackStatistics()
2686 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2687};
2688} // namespace
2689
2690/// ------------------------ Must-Progress Attributes --------------------------
2691namespace {
2692struct AAMustProgressImpl : public AAMustProgress {
2693 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2694 : AAMustProgress(IRP, A) {}
2695
2696 /// See AbstractAttribute::initialize(...).
2697 void initialize(Attributor &A) override {
2698 bool IsKnown;
2699 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2700 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2701 (void)IsKnown;
2702 }
2703
2704 /// See AbstractAttribute::getAsStr()
2705 const std::string getAsStr(Attributor *A) const override {
2706 return getAssumed() ? "mustprogress" : "may-not-progress";
2707 }
2708};
2709
2710struct AAMustProgressFunction final : AAMustProgressImpl {
2711 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2712 : AAMustProgressImpl(IRP, A) {}
2713
2714 /// See AbstractAttribute::updateImpl(...).
2715 ChangeStatus updateImpl(Attributor &A) override {
2716 bool IsKnown;
2717 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2718 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
2719 if (IsKnown)
2720 return indicateOptimisticFixpoint();
2721 return ChangeStatus::UNCHANGED;
2722 }
2723
2724 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2725 IRPosition IPos = IRPosition::callsite_function(*ACS.getInstruction());
2726 bool IsKnownMustProgress;
2727 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2728 A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
2729 /* IgnoreSubsumingPositions */ true);
2730 };
2731
2732 bool AllCallSitesKnown = true;
2733 if (!A.checkForAllCallSites(CheckForMustProgress, *this,
2734 /* RequireAllCallSites */ true,
2735 AllCallSitesKnown))
2736 return indicatePessimisticFixpoint();
2737
2738 return ChangeStatus::UNCHANGED;
2739 }
2740
2741 /// See AbstractAttribute::trackStatistics()
2742 void trackStatistics() const override {
2743 STATS_DECLTRACK_FN_ATTR(mustprogress)
2744 }
2745};
2746
2747/// MustProgress attribute deduction for a call sites.
2748struct AAMustProgressCallSite final : AAMustProgressImpl {
2749 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2750 : AAMustProgressImpl(IRP, A) {}
2751
2752 /// See AbstractAttribute::updateImpl(...).
2753 ChangeStatus updateImpl(Attributor &A) override {
2754 // TODO: Once we have call site specific value information we can provide
2755 // call site specific liveness information and then it makes
2756 // sense to specialize attributes for call sites arguments instead of
2757 // redirecting requests to the callee argument.
2758 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
2759 bool IsKnownMustProgress;
2760 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2761 A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
2762 return indicatePessimisticFixpoint();
2763 return ChangeStatus::UNCHANGED;
2764 }
2765
2766 /// See AbstractAttribute::trackStatistics()
2767 void trackStatistics() const override {
2768 STATS_DECLTRACK_CS_ATTR(mustprogress);
2769 }
2770};
2771} // namespace
2772
2773/// ------------------------ No-Recurse Attributes ----------------------------
2774
2775namespace {
2776struct AANoRecurseImpl : public AANoRecurse {
2777 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2778
2779 /// See AbstractAttribute::initialize(...).
2780 void initialize(Attributor &A) override {
2781 bool IsKnown;
2782 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2783 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2784 (void)IsKnown;
2785 }
2786
2787 /// See AbstractAttribute::getAsStr()
2788 const std::string getAsStr(Attributor *A) const override {
2789 return getAssumed() ? "norecurse" : "may-recurse";
2790 }
2791};
2792
2793struct AANoRecurseFunction final : AANoRecurseImpl {
2794 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2795 : AANoRecurseImpl(IRP, A) {}
2796
2797 /// See AbstractAttribute::updateImpl(...).
2798 ChangeStatus updateImpl(Attributor &A) override {
2799
2800 // If all live call sites are known to be no-recurse, we are as well.
2801 auto CallSitePred = [&](AbstractCallSite ACS) {
2802 bool IsKnownNoRecurse;
2803 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2804 A, this,
2805 IRPosition::function(*ACS.getInstruction()->getFunction()),
2806 DepClassTy::NONE, IsKnownNoRecurse))
2807 return false;
2808 return IsKnownNoRecurse;
2809 };
2810 bool UsedAssumedInformation = false;
2811 if (A.checkForAllCallSites(CallSitePred, *this, true,
2812 UsedAssumedInformation)) {
2813 // If we know all call sites and all are known no-recurse, we are done.
2814 // If all known call sites, which might not be all that exist, are known
2815 // to be no-recurse, we are not done but we can continue to assume
2816 // no-recurse. If one of the call sites we have not visited will become
2817 // live, another update is triggered.
2818 if (!UsedAssumedInformation)
2819 indicateOptimisticFixpoint();
2820 return ChangeStatus::UNCHANGED;
2821 }
2822
2823 const AAInterFnReachability *EdgeReachability =
2824 A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),
2825 DepClassTy::REQUIRED);
2826 if (EdgeReachability && EdgeReachability->canReach(A, *getAnchorScope()))
2827 return indicatePessimisticFixpoint();
2828 return ChangeStatus::UNCHANGED;
2829 }
2830
2831 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2832};
2833
2834/// NoRecurse attribute deduction for a call sites.
2835struct AANoRecurseCallSite final
2836 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2837 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2838 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2839
2840 /// See AbstractAttribute::trackStatistics()
2841 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2842};
2843} // namespace
2844
2845/// ------------------------ No-Convergent Attribute --------------------------
2846
2847namespace {
2848struct AANonConvergentImpl : public AANonConvergent {
2849 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2850 : AANonConvergent(IRP, A) {}
2851
2852 /// See AbstractAttribute::getAsStr()
2853 const std::string getAsStr(Attributor *A) const override {
2854 return getAssumed() ? "non-convergent" : "may-be-convergent";
2855 }
2856};
2857
2858struct AANonConvergentFunction final : AANonConvergentImpl {
2859 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2860 : AANonConvergentImpl(IRP, A) {}
2861
2862 /// See AbstractAttribute::updateImpl(...).
2863 ChangeStatus updateImpl(Attributor &A) override {
2864 // If all function calls are known to not be convergent, we are not
2865 // convergent.
2866 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2867 CallBase &CB = cast<CallBase>(Inst);
2868 auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
2869 if (!Callee || Callee->isIntrinsic()) {
2870 return false;
2871 }
2872 if (Callee->isDeclaration()) {
2873 return !Callee->hasFnAttribute(Attribute::Convergent);
2874 }
2875 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2876 *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
2877 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2878 };
2879
2880 bool UsedAssumedInformation = false;
2881 if (!A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *this,
2882 UsedAssumedInformation)) {
2883 return indicatePessimisticFixpoint();
2884 }
2885 return ChangeStatus::UNCHANGED;
2886 }
2887
2888 ChangeStatus manifest(Attributor &A) override {
2889 if (isKnownNotConvergent() &&
2890 A.hasAttr(getIRPosition(), Attribute::Convergent)) {
2891 A.removeAttrs(getIRPosition(), {Attribute::Convergent});
2892 return ChangeStatus::CHANGED;
2893 }
2894 return ChangeStatus::UNCHANGED;
2895 }
2896
2897 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2898};
2899} // namespace
2900
2901/// -------------------- Undefined-Behavior Attributes ------------------------
2902
2903namespace {
2904struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2905 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2906 : AAUndefinedBehavior(IRP, A) {}
2907
2908 /// See AbstractAttribute::updateImpl(...).
2909 // through a pointer (i.e. also branches etc.)
2910 ChangeStatus updateImpl(Attributor &A) override {
2911 const size_t UBPrevSize = KnownUBInsts.size();
2912 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2913
2914 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2915 // Lang ref now states volatile store is not UB, let's skip them.
2916 if (I.isVolatile() && I.mayWriteToMemory())
2917 return true;
2918
2919 // Skip instructions that are already saved.
2920 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2921 return true;
2922
2923 // If we reach here, we know we have an instruction
2924 // that accesses memory through a pointer operand,
2925 // for which getPointerOperand() should give it to us.
2926 Value *PtrOp =
2927 const_cast<Value *>(getPointerOperand(&I, /* AllowVolatile */ true));
2928 assert(PtrOp &&
2929 "Expected pointer operand of memory accessing instruction");
2930
2931 // Either we stopped and the appropriate action was taken,
2932 // or we got back a simplified value to continue.
2933 std::optional<Value *> SimplifiedPtrOp =
2934 stopOnUndefOrAssumed(A, PtrOp, &I);
2935 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2936 return true;
2937 const Value *PtrOpVal = *SimplifiedPtrOp;
2938
2939 // A memory access through a pointer is considered UB
2940 // only if the pointer has constant null value.
2941 // TODO: Expand it to not only check constant values.
2942 if (!isa<ConstantPointerNull>(PtrOpVal)) {
2943 AssumedNoUBInsts.insert(&I);
2944 return true;
2945 }
2946 const Type *PtrTy = PtrOpVal->getType();
2947
2948 // Because we only consider instructions inside functions,
2949 // assume that a parent function exists.
2950 const Function *F = I.getFunction();
2951
2952 // A memory access using constant null pointer is only considered UB
2953 // if null pointer is _not_ defined for the target platform.
2955 AssumedNoUBInsts.insert(&I);
2956 else
2957 KnownUBInsts.insert(&I);
2958 return true;
2959 };
2960
2961 auto InspectBrInstForUB = [&](Instruction &I) {
2962 // A conditional branch instruction is considered UB if it has `undef`
2963 // condition.
2964
2965 // Skip instructions that are already saved.
2966 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2967 return true;
2968
2969 // We know we have a branch instruction.
2970 auto *BrInst = cast<BranchInst>(&I);
2971
2972 // Unconditional branches are never considered UB.
2973 if (BrInst->isUnconditional())
2974 return true;
2975
2976 // Either we stopped and the appropriate action was taken,
2977 // or we got back a simplified value to continue.
2978 std::optional<Value *> SimplifiedCond =
2979 stopOnUndefOrAssumed(A, BrInst->getCondition(), BrInst);
2980 if (!SimplifiedCond || !*SimplifiedCond)
2981 return true;
2982 AssumedNoUBInsts.insert(&I);
2983 return true;
2984 };
2985
2986 auto InspectCallSiteForUB = [&](Instruction &I) {
2987 // Check whether a callsite always cause UB or not
2988
2989 // Skip instructions that are already saved.
2990 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2991 return true;
2992
2993 // Check nonnull and noundef argument attribute violation for each
2994 // callsite.
2995 CallBase &CB = cast<CallBase>(I);
2996 auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
2997 if (!Callee)
2998 return true;
2999 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3000 // If current argument is known to be simplified to null pointer and the
3001 // corresponding argument position is known to have nonnull attribute,
3002 // the argument is poison. Furthermore, if the argument is poison and
3003 // the position is known to have noundef attriubte, this callsite is
3004 // considered UB.
3005 if (idx >= Callee->arg_size())
3006 break;
3007 Value *ArgVal = CB.getArgOperand(idx);
3008 if (!ArgVal)
3009 continue;
3010 // Here, we handle three cases.
3011 // (1) Not having a value means it is dead. (we can replace the value
3012 // with undef)
3013 // (2) Simplified to undef. The argument violate noundef attriubte.
3014 // (3) Simplified to null pointer where known to be nonnull.
3015 // The argument is a poison value and violate noundef attribute.
3016 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, idx);
3017 bool IsKnownNoUndef;
3018 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3019 A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
3020 if (!IsKnownNoUndef)
3021 continue;
3022 bool UsedAssumedInformation = false;
3023 std::optional<Value *> SimplifiedVal =
3024 A.getAssumedSimplified(IRPosition::value(*ArgVal), *this,
3025 UsedAssumedInformation, AA::Interprocedural);
3026 if (UsedAssumedInformation)
3027 continue;
3028 if (SimplifiedVal && !*SimplifiedVal)
3029 return true;
3030 if (!SimplifiedVal || isa<UndefValue>(**SimplifiedVal)) {
3031 KnownUBInsts.insert(&I);
3032 continue;
3033 }
3034 if (!ArgVal->getType()->isPointerTy() ||
3035 !isa<ConstantPointerNull>(**SimplifiedVal))
3036 continue;
3037 bool IsKnownNonNull;
3038 AA::hasAssumedIRAttr<Attribute::NonNull>(
3039 A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
3040 if (IsKnownNonNull)
3041 KnownUBInsts.insert(&I);
3042 }
3043 return true;
3044 };
3045
3046 auto InspectReturnInstForUB = [&](Instruction &I) {
3047 auto &RI = cast<ReturnInst>(I);
3048 // Either we stopped and the appropriate action was taken,
3049 // or we got back a simplified return value to continue.
3050 std::optional<Value *> SimplifiedRetValue =
3051 stopOnUndefOrAssumed(A, RI.getReturnValue(), &I);
3052 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3053 return true;
3054
3055 // Check if a return instruction always cause UB or not
3056 // Note: It is guaranteed that the returned position of the anchor
3057 // scope has noundef attribute when this is called.
3058 // We also ensure the return position is not "assumed dead"
3059 // because the returned value was then potentially simplified to
3060 // `undef` in AAReturnedValues without removing the `noundef`
3061 // attribute yet.
3062
3063 // When the returned position has noundef attriubte, UB occurs in the
3064 // following cases.
3065 // (1) Returned value is known to be undef.
3066 // (2) The value is known to be a null pointer and the returned
3067 // position has nonnull attribute (because the returned value is
3068 // poison).
3069 if (isa<ConstantPointerNull>(*SimplifiedRetValue)) {
3070 bool IsKnownNonNull;
3071 AA::hasAssumedIRAttr<Attribute::NonNull>(
3072 A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,
3073 IsKnownNonNull);
3074 if (IsKnownNonNull)
3075 KnownUBInsts.insert(&I);
3076 }
3077
3078 return true;
3079 };
3080
3081 bool UsedAssumedInformation = false;
3082 A.checkForAllInstructions(InspectMemAccessInstForUB, *this,
3083 {Instruction::Load, Instruction::Store,
3084 Instruction::AtomicCmpXchg,
3085 Instruction::AtomicRMW},
3086 UsedAssumedInformation,
3087 /* CheckBBLivenessOnly */ true);
3088 A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::Br},
3089 UsedAssumedInformation,
3090 /* CheckBBLivenessOnly */ true);
3091 A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this,
3092 UsedAssumedInformation);
3093
3094 // If the returned position of the anchor scope has noundef attriubte, check
3095 // all returned instructions.
3096 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3097 const IRPosition &ReturnIRP = IRPosition::returned(*getAnchorScope());
3098 if (!A.isAssumedDead(ReturnIRP, this, nullptr, UsedAssumedInformation)) {
3099 bool IsKnownNoUndef;
3100 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3101 A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
3102 if (IsKnownNoUndef)
3103 A.checkForAllInstructions(InspectReturnInstForUB, *this,
3104 {Instruction::Ret}, UsedAssumedInformation,
3105 /* CheckBBLivenessOnly */ true);
3106 }
3107 }
3108
3109 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3110 UBPrevSize != KnownUBInsts.size())
3111 return ChangeStatus::CHANGED;
3112 return ChangeStatus::UNCHANGED;
3113 }
3114
3115 bool isKnownToCauseUB(Instruction *I) const override {
3116 return KnownUBInsts.count(I);
3117 }
3118
3119 bool isAssumedToCauseUB(Instruction *I) const override {
3120 // In simple words, if an instruction is not in the assumed to _not_
3121 // cause UB, then it is assumed UB (that includes those
3122 // in the KnownUBInsts set). The rest is boilerplate
3123 // is to ensure that it is one of the instructions we test
3124 // for UB.
3125
3126 switch (I->getOpcode()) {
3127 case Instruction::Load:
3128 case Instruction::Store:
3129 case Instruction::AtomicCmpXchg:
3130 case Instruction::AtomicRMW:
3131 return !AssumedNoUBInsts.count(I);
3132 case Instruction::Br: {
3133 auto *BrInst = cast<BranchInst>(I);
3134 if (BrInst->isUnconditional())
3135 return false;
3136 return !AssumedNoUBInsts.count(I);
3137 } break;
3138 default:
3139 return false;
3140 }
3141 return false;
3142 }
3143
3144 ChangeStatus manifest(Attributor &A) override {
3145 if (KnownUBInsts.empty())
3146 return ChangeStatus::UNCHANGED;
3147 for (Instruction *I : KnownUBInsts)
3148 A.changeToUnreachableAfterManifest(I);
3149 return ChangeStatus::CHANGED;
3150 }
3151
3152 /// See AbstractAttribute::getAsStr()
3153 const std::string getAsStr(Attributor *A) const override {
3154 return getAssumed() ? "undefined-behavior" : "no-ub";
3155 }
3156
3157 /// Note: The correctness of this analysis depends on the fact that the
3158 /// following 2 sets will stop changing after some point.
3159 /// "Change" here means that their size changes.
3160 /// The size of each set is monotonically increasing
3161 /// (we only add items to them) and it is upper bounded by the number of
3162 /// instructions in the processed function (we can never save more
3163 /// elements in either set than this number). Hence, at some point,
3164 /// they will stop increasing.
3165 /// Consequently, at some point, both sets will have stopped
3166 /// changing, effectively making the analysis reach a fixpoint.
3167
3168 /// Note: These 2 sets are disjoint and an instruction can be considered
3169 /// one of 3 things:
3170 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3171 /// the KnownUBInsts set.
3172 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3173 /// has a reason to assume it).
3174 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3175 /// could not find a reason to assume or prove that it can cause UB,
3176 /// hence it assumes it doesn't. We have a set for these instructions
3177 /// so that we don't reprocess them in every update.
3178 /// Note however that instructions in this set may cause UB.
3179
3180protected:
3181 /// A set of all live instructions _known_ to cause UB.
3182 SmallPtrSet<Instruction *, 8> KnownUBInsts;
3183
3184private:
3185 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3186 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3187
3188 // Should be called on updates in which if we're processing an instruction
3189 // \p I that depends on a value \p V, one of the following has to happen:
3190 // - If the value is assumed, then stop.
3191 // - If the value is known but undef, then consider it UB.
3192 // - Otherwise, do specific processing with the simplified value.
3193 // We return std::nullopt in the first 2 cases to signify that an appropriate
3194 // action was taken and the caller should stop.
3195 // Otherwise, we return the simplified value that the caller should
3196 // use for specific processing.
3197 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3198 Instruction *I) {
3199 bool UsedAssumedInformation = false;
3200 std::optional<Value *> SimplifiedV =
3201 A.getAssumedSimplified(IRPosition::value(*V), *this,
3202 UsedAssumedInformation, AA::Interprocedural);
3203 if (!UsedAssumedInformation) {
3204 // Don't depend on assumed values.
3205 if (!SimplifiedV) {
3206 // If it is known (which we tested above) but it doesn't have a value,
3207 // then we can assume `undef` and hence the instruction is UB.
3208 KnownUBInsts.insert(I);
3209 return std::nullopt;
3210 }
3211 if (!*SimplifiedV)
3212 return nullptr;
3213 V = *SimplifiedV;
3214 }
3215 if (isa<UndefValue>(V)) {
3216 KnownUBInsts.insert(I);
3217 return std::nullopt;
3218 }
3219 return V;
3220 }
3221};
3222
3223struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3224 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3225 : AAUndefinedBehaviorImpl(IRP, A) {}
3226
3227 /// See AbstractAttribute::trackStatistics()
3228 void trackStatistics() const override {
3229 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3230 "Number of instructions known to have UB");
3231 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3232 KnownUBInsts.size();
3233 }
3234};
3235} // namespace
3236
3237/// ------------------------ Will-Return Attributes ----------------------------
3238
3239namespace {
3240// Helper function that checks whether a function has any cycle which we don't
3241// know if it is bounded or not.
3242// Loops with maximum trip count are considered bounded, any other cycle not.
3243static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3244 ScalarEvolution *SE =
3245 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3246 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3247 // If either SCEV or LoopInfo is not available for the function then we assume
3248 // any cycle to be unbounded cycle.
3249 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3250 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3251 if (!SE || !LI) {
3252 for (scc_iterator<Function *> SCCI = scc_begin(&F); !SCCI.isAtEnd(); ++SCCI)
3253 if (SCCI.hasCycle())
3254 return true;
3255 return false;
3256 }
3257
3258 // If there's irreducible control, the function may contain non-loop cycles.
3260 return true;
3261
3262 // Any loop that does not have a max trip count is considered unbounded cycle.
3263 for (auto *L : LI->getLoopsInPreorder()) {
3264 if (!SE->getSmallConstantMaxTripCount(L))
3265 return true;
3266 }
3267 return false;
3268}
3269
3270struct AAWillReturnImpl : public AAWillReturn {
3271 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3272 : AAWillReturn(IRP, A) {}
3273
3274 /// See AbstractAttribute::initialize(...).
3275 void initialize(Attributor &A) override {
3276 bool IsKnown;
3277 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3278 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3279 (void)IsKnown;
3280 }
3281
3282 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3283 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3284 if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
3285 return false;
3286
3287 bool IsKnown;
3288 if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3289 return IsKnown || !KnownOnly;
3290 return false;
3291 }
3292
3293 /// See AbstractAttribute::updateImpl(...).
3294 ChangeStatus updateImpl(Attributor &A) override {
3295 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3296 return ChangeStatus::UNCHANGED;
3297
3298 auto CheckForWillReturn = [&](Instruction &I) {
3299 IRPosition IPos = IRPosition::callsite_function(cast<CallBase>(I));
3300 bool IsKnown;
3301 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3302 A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {
3303 if (IsKnown)
3304 return true;
3305 } else {
3306 return false;
3307 }
3308 bool IsKnownNoRecurse;
3309 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3310 A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
3311 };
3312
3313 bool UsedAssumedInformation = false;
3314 if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,
3315 UsedAssumedInformation))
3316 return indicatePessimisticFixpoint();
3317
3318 return ChangeStatus::UNCHANGED;
3319 }
3320
3321 /// See AbstractAttribute::getAsStr()
3322 const std::string getAsStr(Attributor *A) const override {
3323 return getAssumed() ? "willreturn" : "may-noreturn";
3324 }
3325};
3326
3327struct AAWillReturnFunction final : AAWillReturnImpl {
3328 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3329 : AAWillReturnImpl(IRP, A) {}
3330
3331 /// See AbstractAttribute::initialize(...).
3332 void initialize(Attributor &A) override {
3333 AAWillReturnImpl::initialize(A);
3334
3335 Function *F = getAnchorScope();
3336 assert(F && "Did expect an anchor function");
3337 if (F->isDeclaration() || mayContainUnboundedCycle(*F, A))
3338 indicatePessimisticFixpoint();
3339 }
3340
3341 /// See AbstractAttribute::trackStatistics()
3342 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3343};
3344
3345/// WillReturn attribute deduction for a call sites.
3346struct AAWillReturnCallSite final
3347 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3348 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3349 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3350
3351 /// See AbstractAttribute::updateImpl(...).
3352 ChangeStatus updateImpl(Attributor &A) override {
3353 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3354 return ChangeStatus::UNCHANGED;
3355
3356 return AACalleeToCallSite::updateImpl(A);
3357 }
3358
3359 /// See AbstractAttribute::trackStatistics()
3360 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3361};
3362} // namespace
3363
3364/// -------------------AAIntraFnReachability Attribute--------------------------
3365
3366/// All information associated with a reachability query. This boilerplate code
3367/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3368/// different \p ToTy values.
3369template <typename ToTy> struct ReachabilityQueryInfo {
3370 enum class Reachable {
3371 No,
3372 Yes,
3373 };
3374
3375 /// Start here,
3376 const Instruction *From = nullptr;
3377 /// reach this place,
3378 const ToTy *To = nullptr;
3379 /// without going through any of these instructions,
3380 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3381 /// and remember if it worked:
3382 Reachable Result = Reachable::No;
3383
3384 /// Precomputed hash for this RQI.
3385 unsigned Hash = 0;
3386
3387 unsigned computeHashValue() const {
3388 assert(Hash == 0 && "Computed hash twice!");
3391 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3392 detail::combineHashValue(PairDMI ::getHashValue({From, To}),
3393 InstSetDMI::getHashValue(ExclusionSet));
3394 }
3395
3397 : From(From), To(To) {}
3398
3399 /// Constructor replacement to ensure unique and stable sets are used for the
3400 /// cache.
3402 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3403 : From(&From), To(&To), ExclusionSet(ES) {
3404
3405 if (!ES || ES->empty()) {
3406 ExclusionSet = nullptr;
3407 } else if (MakeUnique) {
3408 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);
3409 }
3410 }
3411
3413 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3414};
3415
3416namespace llvm {
3417template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3420
3423
3424 static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
3426 return &TombstoneKey;
3427 }
3428 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3429 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3430 }
3433 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3434 return false;
3435 return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);
3436 }
3437};
3438
3439#define DefineKeys(ToTy) \
3440 template <> \
3441 ReachabilityQueryInfo<ToTy> \
3442 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
3443 ReachabilityQueryInfo<ToTy>( \
3444 DenseMapInfo<const Instruction *>::getEmptyKey(), \
3445 DenseMapInfo<const ToTy *>::getEmptyKey()); \
3446 template <> \
3447 ReachabilityQueryInfo<ToTy> \
3448 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
3449 ReachabilityQueryInfo<ToTy>( \
3450 DenseMapInfo<const Instruction *>::getTombstoneKey(), \
3451 DenseMapInfo<const ToTy *>::getTombstoneKey());
3452
3454#undef DefineKeys
3455
3456} // namespace llvm
3457
3458namespace {
3459
3460template <typename BaseTy, typename ToTy>
3461struct CachedReachabilityAA : public BaseTy {
3462 using RQITy = ReachabilityQueryInfo<ToTy>;
3463
3464 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3465
3466 /// See AbstractAttribute::isQueryAA.
3467 bool isQueryAA() const override { return true; }
3468
3469 /// See AbstractAttribute::updateImpl(...).
3470 ChangeStatus updateImpl(Attributor &A) override {
3471 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3472 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3473 RQITy *RQI = QueryVector[u];
3474 if (RQI->Result == RQITy::Reachable::No &&
3475 isReachableImpl(A, *RQI, /*IsTemporaryRQI=*/false))
3476 Changed = ChangeStatus::CHANGED;
3477 }
3478 return Changed;
3479 }
3480
3481 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3482 bool IsTemporaryRQI) = 0;
3483
3484 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3485 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3486 RQI.Result = Result;
3487
3488 // Remove the temporary RQI from the cache.
3489 if (IsTemporaryRQI)
3490 QueryCache.erase(&RQI);
3491
3492 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3493 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3494 // this query. 2) We did not use the exclusion set, potentially because
3495 // there is none.
3496 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3497 RQITy PlainRQI(RQI.From, RQI.To);
3498 if (!QueryCache.count(&PlainRQI)) {
3499 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3500 RQIPtr->Result = Result;
3501 QueryVector.push_back(RQIPtr);
3502 QueryCache.insert(RQIPtr);
3503 }
3504 }
3505
3506 // Check if we need to insert a new permanent RQI with the exclusion set.
3507 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3508 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3509 "Did not expect empty set!");
3510 RQITy *RQIPtr = new (A.Allocator)
3511 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3512 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3513 RQIPtr->Result = Result;
3514 assert(!QueryCache.count(RQIPtr));
3515 QueryVector.push_back(RQIPtr);
3516 QueryCache.insert(RQIPtr);
3517 }
3518
3519 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3520 A.registerForUpdate(*this);
3521 return Result == RQITy::Reachable::Yes;
3522 }
3523
3524 const std::string getAsStr(Attributor *A) const override {
3525 // TODO: Return the number of reachable queries.
3526 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3527 }
3528
3529 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3530 typename RQITy::Reachable &Result) {
3531 if (!this->getState().isValidState()) {
3532 Result = RQITy::Reachable::Yes;
3533 return true;
3534 }
3535
3536 // If we have an exclusion set we might be able to find our answer by
3537 // ignoring it first.
3538 if (StackRQI.ExclusionSet) {
3539 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3540 auto It = QueryCache.find(&PlainRQI);
3541 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3542 Result = RQITy::Reachable::No;
3543 return true;
3544 }
3545 }
3546
3547 auto It = QueryCache.find(&StackRQI);
3548 if (It != QueryCache.end()) {
3549 Result = (*It)->Result;
3550 return true;
3551 }
3552
3553 // Insert a temporary for recursive queries. We will replace it with a
3554 // permanent entry later.
3555 QueryCache.insert(&StackRQI);
3556 return false;
3557 }
3558
3559private:
3560 SmallVector<RQITy *> QueryVector;
3561 DenseSet<RQITy *> QueryCache;
3562};
3563
3564struct AAIntraFnReachabilityFunction final
3565 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3566 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3567 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3568 : Base(IRP, A) {
3569 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3570 *IRP.getAssociatedFunction());
3571 }
3572
3573 bool isAssumedReachable(
3574 Attributor &A, const Instruction &From, const Instruction &To,
3575 const AA::InstExclusionSetTy *ExclusionSet) const override {
3576 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3577 if (&From == &To)
3578 return true;
3579
3580 RQITy StackRQI(A, From, To, ExclusionSet, false);
3581 typename RQITy::Reachable Result;
3582 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3583 return NonConstThis->isReachableImpl(A, StackRQI,
3584 /*IsTemporaryRQI=*/true);
3585 return Result == RQITy::Reachable::Yes;
3586 }
3587
3588 ChangeStatus updateImpl(Attributor &A) override {
3589 // We only depend on liveness. DeadEdges is all we care about, check if any
3590 // of them changed.
3591 auto *LivenessAA =
3592 A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
3593 if (LivenessAA &&
3594 llvm::all_of(DeadEdges,
3595 [&](const auto &DeadEdge) {
3596 return LivenessAA->isEdgeDead(DeadEdge.first,
3597 DeadEdge.second);
3598 }) &&
3599 llvm::all_of(DeadBlocks, [&](const BasicBlock *BB) {
3600 return LivenessAA->isAssumedDead(BB);
3601 })) {
3602 return ChangeStatus::UNCHANGED;
3603 }
3604 DeadEdges.clear();
3605 DeadBlocks.clear();
3606 return Base::updateImpl(A);
3607 }
3608
3609 bool isReachableImpl(Attributor &A, RQITy &RQI,
3610 bool IsTemporaryRQI) override {
3611 const Instruction *Origin = RQI.From;
3612 bool UsedExclusionSet = false;
3613
3614 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3615 const AA::InstExclusionSetTy *ExclusionSet) {
3616 const Instruction *IP = &From;
3617 while (IP && IP != &To) {
3618 if (ExclusionSet && IP != Origin && ExclusionSet->count(IP)) {
3619 UsedExclusionSet = true;
3620 break;
3621 }
3622 IP = IP->getNextNode();
3623 }
3624 return IP == &To;
3625 };
3626
3627 const BasicBlock *FromBB = RQI.From->getParent();
3628 const BasicBlock *ToBB = RQI.To->getParent();
3629 assert(FromBB->getParent() == ToBB->getParent() &&
3630 "Not an intra-procedural query!");
3631
3632 // Check intra-block reachability, however, other reaching paths are still
3633 // possible.
3634 if (FromBB == ToBB &&
3635 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3636 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3637 IsTemporaryRQI);
3638
3639 // Check if reaching the ToBB block is sufficient or if even that would not
3640 // ensure reaching the target. In the latter case we are done.
3641 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3642 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3643 IsTemporaryRQI);
3644
3645 const Function *Fn = FromBB->getParent();
3647 if (RQI.ExclusionSet)
3648 for (auto *I : *RQI.ExclusionSet)
3649 if (I->getFunction() == Fn)
3650 ExclusionBlocks.insert(I->getParent());
3651
3652 // Check if we make it out of the FromBB block at all.
3653 if (ExclusionBlocks.count(FromBB) &&
3654 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3655 RQI.ExclusionSet))
3656 return rememberResult(A, RQITy::Reachable::No, RQI, true, IsTemporaryRQI);
3657
3658 auto *LivenessAA =
3659 A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
3660 if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {
3661 DeadBlocks.insert(ToBB);
3662 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3663 IsTemporaryRQI);
3664 }
3665
3668 Worklist.push_back(FromBB);
3669
3671 while (!Worklist.empty()) {
3672 const BasicBlock *BB = Worklist.pop_back_val();
3673 if (!Visited.insert(BB).second)
3674 continue;
3675 for (const BasicBlock *SuccBB : successors(BB)) {
3676 if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {
3677 LocalDeadEdges.insert({BB, SuccBB});
3678 continue;
3679 }
3680 // We checked before if we just need to reach the ToBB block.
3681 if (SuccBB == ToBB)
3682 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3683 IsTemporaryRQI);
3684 if (DT && ExclusionBlocks.empty() && DT->dominates(BB, ToBB))
3685 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3686 IsTemporaryRQI);
3687
3688 if (ExclusionBlocks.count(SuccBB)) {
3689 UsedExclusionSet = true;
3690 continue;
3691 }
3692 Worklist.push_back(SuccBB);
3693 }
3694 }
3695
3696 DeadEdges.insert(LocalDeadEdges.begin(), LocalDeadEdges.end());
3697 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3698 IsTemporaryRQI);
3699 }
3700
3701 /// See AbstractAttribute::trackStatistics()
3702 void trackStatistics() const override {}
3703
3704private:
3705 // Set of assumed dead blocks we used in the last query. If any changes we
3706 // update the state.
3708
3709 // Set of assumed dead edges we used in the last query. If any changes we
3710 // update the state.
3712
3713 /// The dominator tree of the function to short-circuit reasoning.
3714 const DominatorTree *DT = nullptr;
3715};
3716} // namespace
3717
3718/// ------------------------ NoAlias Argument Attribute ------------------------
3719
3721 Attribute::AttrKind ImpliedAttributeKind,
3722 bool IgnoreSubsumingPositions) {
3723 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3724 "Unexpected attribute kind");
3725 Value *Val = &IRP.getAssociatedValue();
3726 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3727 if (isa<AllocaInst>(Val))
3728 return true;
3729 } else {
3730 IgnoreSubsumingPositions = true;
3731 }
3732
3733 if (isa<UndefValue>(Val))
3734 return true;
3735
3736 if (isa<ConstantPointerNull>(Val) &&
3739 return true;
3740
3741 if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
3742 IgnoreSubsumingPositions, Attribute::NoAlias))
3743 return true;
3744
3745 return false;
3746}
3747
3748namespace {
3749struct AANoAliasImpl : AANoAlias {
3750 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3751 assert(getAssociatedType()->isPointerTy() &&
3752 "Noalias is a pointer attribute");
3753 }
3754
3755 const std::string getAsStr(Attributor *A) const override {
3756 return getAssumed() ? "noalias" : "may-alias";
3757 }
3758};
3759
3760/// NoAlias attribute for a floating value.
3761struct AANoAliasFloating final : AANoAliasImpl {
3762 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3763 : AANoAliasImpl(IRP, A) {}
3764
3765 /// See AbstractAttribute::updateImpl(...).
3766 ChangeStatus updateImpl(Attributor &A) override {
3767 // TODO: Implement this.
3768 return indicatePessimisticFixpoint();
3769 }
3770
3771 /// See AbstractAttribute::trackStatistics()
3772 void trackStatistics() const override {
3774 }
3775};
3776
3777/// NoAlias attribute for an argument.
3778struct AANoAliasArgument final
3779 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3780 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3781 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3782
3783 /// See AbstractAttribute::update(...).
3784 ChangeStatus updateImpl(Attributor &A) override {
3785 // We have to make sure no-alias on the argument does not break
3786 // synchronization when this is a callback argument, see also [1] below.
3787 // If synchronization cannot be affected, we delegate to the base updateImpl
3788 // function, otherwise we give up for now.
3789
3790 // If the function is no-sync, no-alias cannot break synchronization.
3791 bool IsKnownNoSycn;
3792 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3793 A, this, IRPosition::function_scope(getIRPosition()),
3794 DepClassTy::OPTIONAL, IsKnownNoSycn))
3795 return Base::updateImpl(A);
3796
3797 // If the argument is read-only, no-alias cannot break synchronization.
3798 bool IsKnown;
3799 if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3800 return Base::updateImpl(A);
3801
3802 // If the argument is never passed through callbacks, no-alias cannot break
3803 // synchronization.
3804 bool UsedAssumedInformation = false;
3805 if (A.checkForAllCallSites(
3806 [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,
3807 true, UsedAssumedInformation))
3808 return Base::updateImpl(A);
3809
3810 // TODO: add no-alias but make sure it doesn't break synchronization by
3811 // introducing fake uses. See:
3812 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3813 // International Workshop on OpenMP 2018,
3814 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3815
3816 return indicatePessimisticFixpoint();
3817 }
3818
3819 /// See AbstractAttribute::trackStatistics()
3820 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3821};
3822
3823struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3824 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3825 : AANoAliasImpl(IRP, A) {}
3826
3827 /// Determine if the underlying value may alias with the call site argument
3828 /// \p OtherArgNo of \p ICS (= the underlying call site).
3829 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3830 const AAMemoryBehavior &MemBehaviorAA,
3831 const CallBase &CB, unsigned OtherArgNo) {
3832 // We do not need to worry about aliasing with the underlying IRP.
3833 if (this->getCalleeArgNo() == (int)OtherArgNo)
3834 return false;
3835
3836 // If it is not a pointer or pointer vector we do not alias.
3837 const Value *ArgOp = CB.getArgOperand(OtherArgNo);
3838 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3839 return false;
3840
3841 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3842 *this, IRPosition::callsite_argument(CB, OtherArgNo), DepClassTy::NONE);
3843
3844 // If the argument is readnone, there is no read-write aliasing.
3845 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3846 A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
3847 return false;
3848 }
3849
3850 // If the argument is readonly and the underlying value is readonly, there
3851 // is no read-write aliasing.
3852 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3853 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3854 IsReadOnly) {
3855 A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3856 A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
3857 return false;
3858 }
3859
3860 // We have to utilize actual alias analysis queries so we need the object.
3861 if (!AAR)
3862 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3863 *getAnchorScope());
3864
3865 // Try to rule it out at the call site.
3866 bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);
3867 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3868 "callsite arguments: "
3869 << getAssociatedValue() << " " << *ArgOp << " => "
3870 << (IsAliasing ? "" : "no-") << "alias \n");
3871
3872 return IsAliasing;
3873 }
3874
3875 bool isKnownNoAliasDueToNoAliasPreservation(
3876 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3877 // We can deduce "noalias" if the following conditions hold.
3878 // (i) Associated value is assumed to be noalias in the definition.
3879 // (ii) Associated value is assumed to be no-capture in all the uses
3880 // possibly executed before this callsite.
3881 // (iii) There is no other pointer argument which could alias with the
3882 // value.
3883
3884 auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
3885 const auto *DerefAA = A.getAAFor<AADereferenceable>(
3886 *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
3887 return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0;
3888 };
3889
3890 const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
3891 const Function *ScopeFn = VIRP.getAnchorScope();
3892 // Check whether the value is captured in the scope using AANoCapture.
3893 // Look at CFG and check only uses possibly executed before this
3894 // callsite.
3895 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3896 Instruction *UserI = cast<Instruction>(U.getUser());
3897
3898 // If UserI is the curr instruction and there is a single potential use of
3899 // the value in UserI we allow the use.
3900 // TODO: We should inspect the operands and allow those that cannot alias
3901 // with the value.
3902 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3903 return true;
3904
3905 if (ScopeFn) {
3906 if (auto *CB = dyn_cast<CallBase>(UserI)) {
3907 if (CB->isArgOperand(&U)) {
3908
3909 unsigned ArgNo = CB->getArgOperandNo(&U);
3910
3911 bool IsKnownNoCapture;
3912 if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
3913 A, this, IRPosition::callsite_argument(*CB, ArgNo),
3914 DepClassTy::OPTIONAL, IsKnownNoCapture))
3915 return true;
3916 }
3917 }
3918
3920 A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,
3921 [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3922 return true;
3923 }
3924
3925 // TODO: We should track the capturing uses in AANoCapture but the problem
3926 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3927 // a value in the module slice.
3928 switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
3929 case UseCaptureKind::NO_CAPTURE:
3930 return true;
3931 case UseCaptureKind::MAY_CAPTURE:
3932 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI
3933 << "\n");
3934 return false;
3935 case UseCaptureKind::PASSTHROUGH:
3936 Follow = true;
3937 return true;
3938 }
3939 llvm_unreachable("unknown UseCaptureKind");
3940 };
3941
3942 bool IsKnownNoCapture;
3943 const AANoCapture *NoCaptureAA = nullptr;
3944 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
3945 A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
3946 if (!IsAssumedNoCapture &&
3947 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3948 if (!A.checkForAllUses(UsePred, *this, getAssociatedValue())) {
3949 LLVM_DEBUG(
3950 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3951 << " cannot be noalias as it is potentially captured\n");
3952 return false;
3953 }
3954 }
3955 if (NoCaptureAA)
3956 A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);
3957
3958 // Check there is no other pointer argument which could alias with the
3959 // value passed at this call site.
3960 // TODO: AbstractCallSite
3961 const auto &CB = cast<CallBase>(getAnchorValue());
3962 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3963 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
3964 return false;
3965
3966 return true;
3967 }
3968
3969 /// See AbstractAttribute::updateImpl(...).
3970 ChangeStatus updateImpl(Attributor &A) override {
3971 // If the argument is readnone we are done as there are no accesses via the
3972 // argument.
3973 auto *MemBehaviorAA =
3974 A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
3975 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
3976 A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3977 return ChangeStatus::UNCHANGED;
3978 }
3979
3980 bool IsKnownNoAlias;
3981 const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
3982 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
3983 A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
3984 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
3985 << " is not no-alias at the definition\n");
3986 return indicatePessimisticFixpoint();
3987 }
3988
3989 AAResults *AAR = nullptr;
3990 if (MemBehaviorAA &&
3991 isKnownNoAliasDueToNoAliasPreservation(A, AAR, *MemBehaviorAA)) {
3992 LLVM_DEBUG(
3993 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
3994 return ChangeStatus::UNCHANGED;
3995 }
3996
3997 return indicatePessimisticFixpoint();
3998 }
3999
4000 /// See AbstractAttribute::trackStatistics()
4001 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4002};
4003
4004/// NoAlias attribute for function return value.
4005struct AANoAliasReturned final : AANoAliasImpl {
4006 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4007 : AANoAliasImpl(IRP, A) {}
4008
4009 /// See AbstractAttribute::updateImpl(...).
4010 ChangeStatus updateImpl(Attributor &A) override {
4011
4012 auto CheckReturnValue = [&](Value &RV) -> bool {
4013 if (Constant *C = dyn_cast<Constant>(&RV))
4014 if (C->isNullValue() || isa<UndefValue>(C))
4015 return true;
4016
4017 /// For now, we can only deduce noalias if we have call sites.
4018 /// FIXME: add more support.
4019 if (!isa<CallBase>(&RV))
4020 return false;
4021
4022 const IRPosition &RVPos = IRPosition::value(RV);
4023 bool IsKnownNoAlias;
4024 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4025 A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
4026 return false;
4027
4028 bool IsKnownNoCapture;
4029 const AANoCapture *NoCaptureAA = nullptr;
4030 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
4031 A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
4032 &NoCaptureAA);
4033 return IsAssumedNoCapture ||
4034 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4035 };
4036
4037 if (!A.checkForAllReturnedValues(CheckReturnValue, *this))
4038 return indicatePessimisticFixpoint();
4039
4040 return ChangeStatus::UNCHANGED;
4041 }
4042
4043 /// See AbstractAttribute::trackStatistics()
4044 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4045};
4046
4047/// NoAlias attribute deduction for a call site return value.
4048struct AANoAliasCallSiteReturned final
4049 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4050 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4051 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4052
4053 /// See AbstractAttribute::trackStatistics()
4054 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4055};
4056} // namespace
4057
4058/// -------------------AAIsDead Function Attribute-----------------------
4059
4060namespace {
4061struct AAIsDeadValueImpl : public AAIsDead {
4062 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4063
4064 /// See AAIsDead::isAssumedDead().
4065 bool isAssumedDead() const override { return isAssumed(IS_DEAD); }
4066
4067 /// See AAIsDead::isKnownDead().
4068 bool isKnownDead() const override { return isKnown(IS_DEAD); }
4069
4070 /// See AAIsDead::isAssumedDead(BasicBlock *).
4071 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4072
4073 /// See AAIsDead::isKnownDead(BasicBlock *).
4074 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4075
4076 /// See AAIsDead::isAssumedDead(Instruction *I).
4077 bool isAssumedDead(const Instruction *I) const override {
4078 return I == getCtxI() && isAssumedDead();
4079 }
4080
4081 /// See AAIsDead::isKnownDead(Instruction *I).
4082 bool isKnownDead(const Instruction *I) const override {
4083 return isAssumedDead(I) && isKnownDead();
4084 }
4085
4086 /// See AbstractAttribute::getAsStr().
4087 const std::string getAsStr(Attributor *A) const override {
4088 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4089 }
4090
4091 /// Check if all uses are assumed dead.
4092 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4093 // Callers might not check the type, void has no uses.
4094 if (V.getType()->isVoidTy() || V.use_empty())
4095 return true;
4096
4097 // If we replace a value with a constant there are no uses left afterwards.
4098 if (!isa<Constant>(V)) {
4099 if (auto *I = dyn_cast<Instruction>(&V))
4100 if (!A.isRunOn(*I->getFunction()))
4101 return false;
4102 bool UsedAssumedInformation = false;
4103 std::optional<Constant *> C =
4104 A.getAssumedConstant(V, *this, UsedAssumedInformation);
4105 if (!C || *C)
4106 return true;
4107 }
4108
4109 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4110 // Explicitly set the dependence class to required because we want a long
4111 // chain of N dependent instructions to be considered live as soon as one is
4112 // without going through N update cycles. This is not required for
4113 // correctness.
4114 return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false,
4115 DepClassTy::REQUIRED,
4116 /* IgnoreDroppableUses */ false);
4117 }
4118
4119 /// Determine if \p I is assumed to be side-effect free.
4120 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4122 return true;
4123
4124 auto *CB = dyn_cast<CallBase>(I);
4125 if (!CB || isa<IntrinsicInst>(CB))
4126 return false;
4127
4128 const IRPosition &CallIRP = IRPosition::callsite_function(*CB);
4129
4130 bool IsKnownNoUnwind;
4131 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4132 A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
4133 return false;
4134
4135 bool IsKnown;
4136 return AA::isAssumedReadOnly(A, CallIRP, *this, IsKnown);
4137 }
4138};
4139
4140struct AAIsDeadFloating : public AAIsDeadValueImpl {
4141 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4142 : AAIsDeadValueImpl(IRP, A) {}
4143
4144 /// See AbstractAttribute::initialize(...).
4145 void initialize(Attributor &A) override {
4146 AAIsDeadValueImpl::initialize(A);
4147
4148 if (isa<UndefValue>(getAssociatedValue())) {
4149 indicatePessimisticFixpoint();
4150 return;
4151 }
4152
4153 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4154 if (!isAssumedSideEffectFree(A, I)) {
4155 if (!isa_and_nonnull<StoreInst>(I) && !isa_and_nonnull<FenceInst>(I))
4156 indicatePessimisticFixpoint();
4157 else
4158 removeAssumedBits(HAS_NO_EFFECT);
4159 }
4160 }
4161
4162 bool isDeadFence(Attributor &A, FenceInst &FI) {
4163 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4164 IRPosition::function(*FI.getFunction()), *this, DepClassTy::NONE);
4165 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4166 return false;
4167 A.recordDependence(*ExecDomainAA, *this, DepClassTy::OPTIONAL);
4168 return true;
4169 }
4170
4171 bool isDeadStore(Attributor &A, StoreInst &SI,
4172 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4173 // Lang ref now states volatile store is not UB/dead, let's skip them.
4174 if (SI.isVolatile())
4175 return false;
4176
4177 // If we are collecting assumes to be deleted we are in the manifest stage.
4178 // It's problematic to collect the potential copies again now so we use the
4179 // cached ones.
4180 bool UsedAssumedInformation = false;
4181 if (!AssumeOnlyInst) {
4182 PotentialCopies.clear();
4183 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, *this,
4184 UsedAssumedInformation)) {
4185 LLVM_DEBUG(
4186 dbgs()
4187 << "[AAIsDead] Could not determine potential copies of store!\n");
4188 return false;
4189 }
4190 }
4191 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4192 << " potential copies.\n");
4193
4194 InformationCache &InfoCache = A.getInfoCache();
4195 return llvm::all_of(PotentialCopies, [&](Value *V) {
4196 if (A.isAssumedDead(IRPosition::value(*V), this, nullptr,
4197 UsedAssumedInformation))
4198 return true;
4199 if (auto *LI = dyn_cast<LoadInst>(V)) {
4200 if (llvm::all_of(LI->uses(), [&](const Use &U) {
4201 auto &UserI = cast<Instruction>(*U.getUser());
4202 if (InfoCache.isOnlyUsedByAssume(UserI)) {
4203 if (AssumeOnlyInst)
4204 AssumeOnlyInst->insert(&UserI);
4205 return true;
4206 }
4207 return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation);
4208 })) {
4209 return true;
4210 }
4211 }
4212 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4213 << " is assumed live!\n");
4214 return false;
4215 });
4216 }
4217
4218 /// See AbstractAttribute::getAsStr().
4219 const std::string getAsStr(Attributor *A) const override {
4220 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4221 if (isa_and_nonnull<StoreInst>(I))
4222 if (isValidState())
4223 return "assumed-dead-store";
4224 if (isa_and_nonnull<FenceInst>(I))
4225 if (isValidState())
4226 return "assumed-dead-fence";
4227 return AAIsDeadValueImpl::getAsStr(A);
4228 }
4229
4230 /// See AbstractAttribute::updateImpl(...).
4231 ChangeStatus updateImpl(Attributor &A) override {
4232 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4233 if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
4234 if (!isDeadStore(A, *SI))
4235 return indicatePessimisticFixpoint();
4236 } else if (auto *FI = dyn_cast_or_null<FenceInst>(I)) {
4237 if (!isDeadFence(A, *FI))
4238 return indicatePessimisticFixpoint();
4239 } else {
4240 if (!isAssumedSideEffectFree(A, I))
4241 return indicatePessimisticFixpoint();
4242 if (!areAllUsesAssumedDead(A, getAssociatedValue()))
4243 return indicatePessimisticFixpoint();
4244 }
4246 }
4247
4248 bool isRemovableStore() const override {
4249 return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue());
4250 }
4251
4252 /// See AbstractAttribute::manifest(...).
4253 ChangeStatus manifest(Attributor &A) override {
4254 Value &V = getAssociatedValue();
4255 if (auto *I = dyn_cast<Instruction>(&V)) {
4256 // If we get here we basically know the users are all dead. We check if
4257 // isAssumedSideEffectFree returns true here again because it might not be
4258 // the case and only the users are dead but the instruction (=call) is
4259 // still needed.
4260 if (auto *SI = dyn_cast<StoreInst>(I)) {
4261 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4262 bool IsDead = isDeadStore(A, *SI, &AssumeOnlyInst);
4263 (void)IsDead;
4264 assert(IsDead && "Store was assumed to be dead!");
4265 A.deleteAfterManifest(*I);
4266 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4267 Instruction *AOI = AssumeOnlyInst[i];
4268 for (auto *Usr : AOI->users())
4269 AssumeOnlyInst.insert(cast<Instruction>(Usr));
4270 A.deleteAfterManifest(*AOI);
4271 }
4272 return ChangeStatus::CHANGED;
4273 }
4274 if (auto *FI = dyn_cast<FenceInst>(I)) {
4275 assert(isDeadFence(A, *FI));
4276 A.deleteAfterManifest(*FI);
4277 return ChangeStatus::CHANGED;
4278 }
4279 if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(I)) {
4280 A.deleteAfterManifest(*I);
4281 return ChangeStatus::CHANGED;
4282 }
4283 }
4285 }
4286
4287 /// See AbstractAttribute::trackStatistics()
4288 void trackStatistics() const override {
4290 }
4291
4292private:
4293 // The potential copies of a dead store, used for deletion during manifest.
4294 SmallSetVector<Value *, 4> PotentialCopies;
4295};
4296
4297struct AAIsDeadArgument : public AAIsDeadFloating {
4298 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4299 : AAIsDeadFloating(IRP, A) {}
4300
4301 /// See AbstractAttribute::manifest(...).
4302 ChangeStatus manifest(Attributor &A) override {
4303 Argument &Arg = *getAssociatedArgument();
4304 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4305 if (A.registerFunctionSignatureRewrite(
4306 Arg, /* ReplacementTypes */ {},
4309 return ChangeStatus::CHANGED;
4310 }
4311 return ChangeStatus::UNCHANGED;
4312 }
4313
4314 /// See AbstractAttribute::trackStatistics()
4315 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4316};
4317
4318struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4319 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4320 : AAIsDeadValueImpl(IRP, A) {}
4321
4322 /// See AbstractAttribute::initialize(...).
4323 void initialize(Attributor &A) override {
4324 AAIsDeadValueImpl::initialize(A);
4325 if (isa<UndefValue>(getAssociatedValue()))
4326 indicatePessimisticFixpoint();
4327 }
4328
4329 /// See AbstractAttribute::updateImpl(...).
4330 ChangeStatus updateImpl(Attributor &A) override {
4331 // TODO: Once we have call site specific value information we can provide
4332 // call site specific liveness information and then it makes
4333 // sense to specialize attributes for call sites arguments instead of
4334 // redirecting requests to the callee argument.
4335 Argument *Arg = getAssociatedArgument();
4336 if (!Arg)
4337 return indicatePessimisticFixpoint();
4338 const IRPosition &ArgPos = IRPosition::argument(*Arg);
4339 auto *ArgAA = A.getAAFor<AAIsDead>(*this, ArgPos, DepClassTy::REQUIRED);
4340 if (!ArgAA)
4341 return indicatePessimisticFixpoint();
4342 return clampStateAndIndicateChange(getState(), ArgAA->getState());
4343 }
4344
4345 /// See AbstractAttribute::manifest(...).
4346 ChangeStatus manifest(Attributor &A) override {
4347 CallBase &CB = cast<CallBase>(getAnchorValue());
4348 Use &U = CB.getArgOperandUse(getCallSiteArgNo());
4349 assert(!isa<UndefValue>(U.get()) &&
4350 "Expected undef values to be filtered out!");
4351 UndefValue &UV = *UndefValue::get(U->getType());
4352 if (A.changeUseAfterManifest(U, UV))
4353 return ChangeStatus::CHANGED;
4354 return ChangeStatus::UNCHANGED;
4355 }
4356
4357 /// See AbstractAttribute::trackStatistics()
4358 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4359};
4360
4361struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4362 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4363 : AAIsDeadFloating(IRP, A) {}
4364
4365 /// See AAIsDead::isAssumedDead().
4366 bool isAssumedDead() const override {
4367 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4368 }
4369
4370 /// See AbstractAttribute::initialize(...).
4371 void initialize(Attributor &A) override {
4372 AAIsDeadFloating::initialize(A);
4373 if (isa<UndefValue>(getAssociatedValue())) {
4374 indicatePessimisticFixpoint();
4375 return;
4376 }
4377
4378 // We track this separately as a secondary state.
4379 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, getCtxI());
4380 }
4381
4382 /// See AbstractAttribute::updateImpl(...).
4383 ChangeStatus updateImpl(Attributor &A) override {
4384 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4385 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, getCtxI())) {
4386 IsAssumedSideEffectFree = false;
4387 Changed = ChangeStatus::CHANGED;
4388 }
4389 if (!areAllUsesAssumedDead(A, getAssociatedValue()))
4390 return indicatePessimisticFixpoint();
4391 return Changed;
4392 }
4393
4394 /// See AbstractAttribute::trackStatistics()
4395 void trackStatistics() const override {
4396 if (IsAssumedSideEffectFree)
4398 else
4399 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4400 }
4401
4402 /// See AbstractAttribute::getAsStr().
4403 const std::string getAsStr(Attributor *A) const override {
4404 return isAssumedDead()
4405 ? "assumed-dead"
4406 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4407 }
4408
4409private:
4410 bool IsAssumedSideEffectFree = true;
4411};
4412
4413struct AAIsDeadReturned : public AAIsDeadValueImpl {
4414 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4415 : AAIsDeadValueImpl(IRP, A) {}
4416
4417 /// See AbstractAttribute::updateImpl(...).
4418 ChangeStatus updateImpl(Attributor &A) override {
4419
4420 bool UsedAssumedInformation = false;
4421 A.checkForAllInstructions([](Instruction &) { return true; }, *this,
4422 {Instruction::Ret}, UsedAssumedInformation);
4423
4424 auto PredForCallSite = [&](AbstractCallSite ACS) {
4425 if (ACS.isCallbackCall() || !ACS.getInstruction())
4426 return false;
4427 return areAllUsesAssumedDead(A, *ACS.getInstruction());
4428 };
4429
4430 if (!A.checkForAllCallSites(PredForCallSite, *this, true,
4431 UsedAssumedInformation))
4432 return indicatePessimisticFixpoint();
4433
4434 return ChangeStatus::UNCHANGED;
4435 }
4436
4437 /// See AbstractAttribute::manifest(...).
4438 ChangeStatus manifest(Attributor &A) override {
4439 // TODO: Rewrite the signature to return void?
4440 bool AnyChange = false;
4441 UndefValue &UV = *UndefValue::get(getAssociatedFunction()->getReturnType());
4442 auto RetInstPred = [&](Instruction &I) {
4443 ReturnInst &RI = cast<ReturnInst>(I);
4444 if (!isa<UndefValue>(RI.getReturnValue()))
4445 AnyChange |= A.changeUseAfterManifest(RI.getOperandUse(0), UV);
4446 return true;
4447 };
4448 bool UsedAssumedInformation = false;
4449 A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
4450 UsedAssumedInformation);
4451 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4452 }
4453
4454 /// See AbstractAttribute::trackStatistics()
4455 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4456};
4457
4458struct AAIsDeadFunction : public AAIsDead {
4459 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4460
4461 /// See AbstractAttribute::initialize(...).
4462 void initialize(Attributor &A) override {
4463 Function *F = getAnchorScope();
4464 assert(F && "Did expect an anchor function");
4465 if (!isAssumedDeadInternalFunction(A)) {
4466 ToBeExploredFrom.insert(&F->getEntryBlock().front());
4467 assumeLive(A, F->getEntryBlock());
4468 }
4469 }
4470
4471 bool isAssumedDeadInternalFunction(Attributor &A) {
4472 if (!getAnchorScope()->hasLocalLinkage())
4473 return false;
4474 bool UsedAssumedInformation = false;
4475 return A.checkForAllCallSites([](AbstractCallSite) { return false; }, *this,
4476 true, UsedAssumedInformation);
4477 }
4478
4479 /// See AbstractAttribute::getAsStr().
4480 const std::string getAsStr(Attributor *A) const override {
4481 return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) + "/" +
4482 std::to_string(getAnchorScope()->size()) + "][#TBEP " +
4483 std::to_string(ToBeExploredFrom.size()) + "][#KDE " +
4484 std::to_string(KnownDeadEnds.size()) + "]";
4485 }
4486
4487 /// See AbstractAttribute::manifest(...).
4488 ChangeStatus manifest(Attributor &A) override {
4489 assert(getState().isValidState() &&
4490 "Attempted to manifest an invalid state!");
4491
4492 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4493 Function &F = *getAnchorScope();
4494
4495 if (AssumedLiveBlocks.empty()) {
4496 A.deleteAfterManifest(F);
4497 return ChangeStatus::CHANGED;
4498 }
4499
4500 // Flag to determine if we can change an invoke to a call assuming the
4501 // callee is nounwind. This is not possible if the personality of the
4502 // function allows to catch asynchronous exceptions.
4503 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4504
4505 KnownDeadEnds.set_union(ToBeExploredFrom);
4506 for (const Instruction *DeadEndI : KnownDeadEnds) {
4507 auto *CB = dyn_cast<CallBase>(DeadEndI);
4508 if (!CB)
4509 continue;
4510 bool IsKnownNoReturn;
4511 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4512 A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,
4513 IsKnownNoReturn);
4514 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(CB)))
4515 continue;
4516
4517 if (auto *II = dyn_cast<InvokeInst>(DeadEndI))
4518 A.registerInvokeWithDeadSuccessor(const_cast<InvokeInst &>(*II));
4519 else
4520 A.changeToUnreachableAfterManifest(
4521 const_cast<Instruction *>(DeadEndI->getNextNode()));
4522 HasChanged = ChangeStatus::CHANGED;
4523 }
4524
4525 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4526 for (BasicBlock &BB : F)
4527 if (!AssumedLiveBlocks.count(&BB)) {
4528 A.deleteAfterManifest(BB);
4530 HasChanged = ChangeStatus::CHANGED;
4531 }
4532
4533 return HasChanged;
4534 }
4535
4536 /// See AbstractAttribute::updateImpl(...).
4537 ChangeStatus updateImpl(Attributor &A) override;
4538
4539 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4540 assert(From->getParent() == getAnchorScope() &&
4541 To->getParent() == getAnchorScope() &&
4542 "Used AAIsDead of the wrong function");
4543 return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));
4544 }
4545
4546 /// See AbstractAttribute::trackStatistics()
4547 void trackStatistics() const override {}
4548
4549 /// Returns true if the function is assumed dead.
4550 bool isAssumedDead() const override { return false; }
4551
4552 /// See AAIsDead::isKnownDead().
4553 bool isKnownDead() const override { return false; }
4554
4555 /// See AAIsDead::isAssumedDead(BasicBlock *).
4556 bool isAssumedDead(const BasicBlock *BB) const override {
4557 assert(BB->getParent() == getAnchorScope() &&
4558 "BB must be in the same anchor scope function.");
4559
4560 if (!getAssumed())
4561 return false;
4562 return !AssumedLiveBlocks.count(BB);
4563 }
4564
4565 /// See AAIsDead::isKnownDead(BasicBlock *).
4566 bool isKnownDead(const BasicBlock *BB) const override {
4567 return getKnown() && isAssumedDead(BB);
4568 }
4569
4570 /// See AAIsDead::isAssumed(Instruction *I).
4571 bool isAssumedDead(const Instruction *I) const override {
4572 assert(I->getParent()->getParent() == getAnchorScope() &&
4573 "Instruction must be in the same anchor scope function.");
4574
4575 if (!getAssumed())
4576 return false;
4577
4578 // If it is not in AssumedLiveBlocks then it for sure dead.
4579 // Otherwise, it can still be after noreturn call in a live block.
4580 if (!AssumedLiveBlocks.count(I->getParent()))
4581 return true;
4582
4583 // If it is not after a liveness barrier it is live.
4584 const Instruction *PrevI = I->getPrevNode();
4585 while (PrevI) {
4586 if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))
4587 return true;
4588 PrevI = PrevI->getPrevNode();
4589 }
4590 return false;
4591 }
4592
4593 /// See AAIsDead::isKnownDead(Instruction *I).
4594 bool isKnownDead(const Instruction *I) const override {
4595 return getKnown() && isAssumedDead(I);
4596 }
4597
4598 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4599 /// that internal function called from \p BB should now be looked at.
4600 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4601 if (!AssumedLiveBlocks.insert(&BB).second)
4602 return false;
4603
4604 // We assume that all of BB is (probably) live now and if there are calls to
4605 // internal functions we will assume that those are now live as well. This
4606 // is a performance optimization for blocks with calls to a lot of internal
4607 // functions. It can however cause dead functions to be treated as live.
4608 for (const Instruction &I : BB)
4609 if (const auto *CB = dyn_cast<CallBase>(&I))
4610 if (auto *F = dyn_cast_if_present<Function>(CB->getCalledOperand()))
4611 if (F->hasLocalLinkage())
4612 A.markLiveInternalFunction(*F);
4613 return true;
4614 }
4615
4616 /// Collection of instructions that need to be explored again, e.g., we
4617 /// did assume they do not transfer control to (one of their) successors.
4619
4620 /// Collection of instructions that are known to not transfer control.
4622
4623 /// Collection of all assumed live edges
4625
4626 /// Collection of all assumed live BasicBlocks.
4627 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4628};
4629
4630static bool
4631identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4633 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4634 const IRPosition &IPos = IRPosition::callsite_function(CB);
4635
4636 bool IsKnownNoReturn;
4637 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4638 A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))
4639 return !IsKnownNoReturn;
4640 if (CB.isTerminator())
4641 AliveSuccessors.push_back(&CB.getSuccessor(0)->front());
4642 else
4643 AliveSuccessors.push_back(CB.getNextNode());
4644 return false;
4645}
4646
4647static bool
4648identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4650 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4651 bool UsedAssumedInformation =
4652 identifyAliveSuccessors(A, cast<CallBase>(II), AA, AliveSuccessors);
4653
4654 // First, determine if we can change an invoke to a call assuming the
4655 // callee is nounwind. This is not possible if the personality of the
4656 // function allows to catch asynchronous exceptions.
4657 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*II.getFunction())) {
4658 AliveSuccessors.push_back(&II.getUnwindDest()->front());
4659 } else {
4660 const IRPosition &IPos = IRPosition::callsite_function(II);
4661
4662 bool IsKnownNoUnwind;
4663 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4664 A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
4665 UsedAssumedInformation |= !IsKnownNoUnwind;
4666 } else {
4667 AliveSuccessors.push_back(&II.getUnwindDest()->front());
4668 }
4669 }
4670 return UsedAssumedInformation;
4671}
4672
4673static bool
4674identifyAliveSuccessors(Attributor &A, const BranchInst &BI,
4676 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4677 bool UsedAssumedInformation = false;
4678 if (BI.getNumSuccessors() == 1) {
4679 AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
4680 } else {
4681 std::optional<Constant *> C =
4682 A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation);
4683 if (!C || isa_and_nonnull<UndefValue>(*C)) {
4684 // No value yet, assume both edges are dead.
4685 } else if (isa_and_nonnull<ConstantInt>(*C)) {
4686 const BasicBlock *SuccBB =
4687 BI.getSuccessor(1 - cast<ConstantInt>(*C)->getValue().getZExtValue());
4688 AliveSuccessors.push_back(&SuccBB->front());
4689 } else {
4690 AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
4691 AliveSuccessors.push_back(&BI.getSuccessor(1)->front());
4692 UsedAssumedInformation = false;
4693 }
4694 }
4695 return UsedAssumedInformation;
4696}
4697
4698static bool
4699identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4701 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4702 bool UsedAssumedInformation = false;
4704 if (!A.getAssumedSimplifiedValues(IRPosition::value(*SI.getCondition()), &AA,
4705 Values, AA::AnyScope,
4706 UsedAssumedInformation)) {
4707 // Something went wrong, assume all successors are live.
4708 for (const BasicBlock *SuccBB : successors(SI.getParent()))
4709 AliveSuccessors.push_back(&SuccBB->front());
4710 return false;
4711 }
4712
4713 if (Values.empty() ||
4714 (Values.size() == 1 &&
4715 isa_and_nonnull<UndefValue>(Values.front().getValue()))) {
4716 // No valid value yet, assume all edges are dead.
4717 return UsedAssumedInformation;
4718 }
4719
4720 Type &Ty = *SI.getCondition()->getType();
4722 auto CheckForConstantInt = [&](Value *V) {
4723 if (auto *CI = dyn_cast_if_present<ConstantInt>(AA::getWithType(*V, Ty))) {
4724 Constants.insert(CI);
4725 return true;
4726 }
4727 return false;
4728 };
4729
4730 if (!all_of(Values, [&](AA::ValueAndContext &VAC) {
4731 return CheckForConstantInt(VAC.getValue());
4732 })) {
4733 for (const BasicBlock *SuccBB : successors(SI.getParent()))
4734 AliveSuccessors.push_back(&SuccBB->front());
4735 return UsedAssumedInformation;
4736 }
4737
4738 unsigned MatchedCases = 0;
4739 for (const auto &CaseIt : SI.cases()) {
4740 if (Constants.count(CaseIt.getCaseValue())) {
4741 ++MatchedCases;
4742 AliveSuccessors.push_back(&CaseIt.getCaseSuccessor()->front());
4743 }
4744 }
4745
4746 // If all potential values have been matched, we will not visit the default
4747 // case.
4748 if (MatchedCases < Constants.size())
4749 AliveSuccessors.push_back(&SI.getDefaultDest()->front());
4750 return UsedAssumedInformation;
4751}
4752
4753ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4755
4756 if (AssumedLiveBlocks.empty()) {
4757 if (isAssumedDeadInternalFunction(A))
4759
4760 Function *F = getAnchorScope();
4761 ToBeExploredFrom.insert(&F->getEntryBlock().front());
4762 assumeLive(A, F->getEntryBlock());
4763 Change = ChangeStatus::CHANGED;
4764 }
4765
4766 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4767 << getAnchorScope()->size() << "] BBs and "
4768 << ToBeExploredFrom.size() << " exploration points and "
4769 << KnownDeadEnds.size() << " known dead ends\n");
4770
4771 // Copy and clear the list of instructions we need to explore from. It is
4772 // refilled with instructions the next update has to look at.
4773 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4774 ToBeExploredFrom.end());
4775 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4776
4778 while (!Worklist.empty()) {
4779 const Instruction *I = Worklist.pop_back_val();
4780 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4781
4782 // Fast forward for uninteresting instructions. We could look for UB here
4783 // though.
4784 while (!I->isTerminator() && !isa<CallBase>(I))
4785 I = I->getNextNode();
4786
4787 AliveSuccessors.clear();
4788
4789 bool UsedAssumedInformation = false;
4790 switch (I->getOpcode()) {
4791 // TODO: look for (assumed) UB to backwards propagate "deadness".
4792 default:
4793 assert(I->isTerminator() &&
4794 "Expected non-terminators to be handled already!");
4795 for (const BasicBlock *SuccBB : successors(I->getParent()))
4796 AliveSuccessors.push_back(&SuccBB->front());
4797 break;
4798 case Instruction::Call:
4799 UsedAssumedInformation = identifyAliveSuccessors(A, cast<CallInst>(*I),
4800 *this, AliveSuccessors);
4801 break;
4802 case Instruction::Invoke:
4803 UsedAssumedInformation = identifyAliveSuccessors(A, cast<InvokeInst>(*I),
4804 *this, AliveSuccessors);
4805 break;
4806 case Instruction::Br:
4807 UsedAssumedInformation = identifyAliveSuccessors(A, cast<BranchInst>(*I),
4808 *this, AliveSuccessors);
4809 break;
4810 case Instruction::Switch:
4811 UsedAssumedInformation = identifyAliveSuccessors(A, cast<SwitchInst>(*I),
4812 *this, AliveSuccessors);
4813 break;
4814 }
4815
4816 if (UsedAssumedInformation) {
4817 NewToBeExploredFrom.insert(I);
4818 } else if (AliveSuccessors.empty() ||
4819 (I->isTerminator() &&
4820 AliveSuccessors.size() < I->getNumSuccessors())) {
4821 if (KnownDeadEnds.insert(I))
4822 Change = ChangeStatus::CHANGED;
4823 }
4824
4825 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4826 << AliveSuccessors.size() << " UsedAssumedInformation: "
4827 << UsedAssumedInformation << "\n");
4828
4829 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4830 if (!I->isTerminator()) {
4831 assert(AliveSuccessors.size() == 1 &&
4832 "Non-terminator expected to have a single successor!");
4833 Worklist.push_back(AliveSuccessor);
4834 } else {
4835 // record the assumed live edge
4836 auto Edge = std::make_pair(I->getParent(), AliveSuccessor->getParent());
4837 if (AssumedLiveEdges.insert(Edge).second)
4838 Change = ChangeStatus::CHANGED;
4839 if (assumeLive(A, *AliveSuccessor->getParent()))
4840 Worklist.push_back(AliveSuccessor);
4841 }
4842 }
4843 }
4844
4845 // Check if the content of ToBeExploredFrom changed, ignore the order.
4846 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4847 llvm::any_of(NewToBeExploredFrom, [&](const Instruction *I) {
4848 return !ToBeExploredFrom.count(I);
4849 })) {
4850 Change = ChangeStatus::CHANGED;
4851 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4852 }
4853
4854 // If we know everything is live there is no need to query for liveness.
4855 // Instead, indicating a pessimistic fixpoint will cause the state to be
4856 // "invalid" and all queries to be answered conservatively without lookups.
4857 // To be in this state we have to (1) finished the exploration and (3) not
4858 // discovered any non-trivial dead end and (2) not ruled unreachable code
4859 // dead.
4860 if (ToBeExploredFrom.empty() &&
4861 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4862 llvm::all_of(KnownDeadEnds, [](const Instruction *DeadEndI) {
4863 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4864 }))
4865 return indicatePessimisticFixpoint();
4866 return Change;
4867}
4868
4869/// Liveness information for a call sites.
4870struct AAIsDeadCallSite final : AAIsDeadFunction {
4871 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4872 : AAIsDeadFunction(IRP, A) {}
4873
4874 /// See AbstractAttribute::initialize(...).
4875 void initialize(Attributor &A) override {
4876 // TODO: Once we have call site specific value information we can provide
4877 // call site specific liveness information and then it makes
4878 // sense to specialize attributes for call sites instead of
4879 // redirecting requests to the callee.
4880 llvm_unreachable("Abstract attributes for liveness are not "
4881 "supported for call sites yet!");
4882 }
4883
4884 /// See AbstractAttribute::updateImpl(...).
4885 ChangeStatus updateImpl(Attributor &A) override {
4886 return indicatePessimisticFixpoint();
4887 }
4888
4889 /// See AbstractAttribute::trackStatistics()
4890 void trackStatistics() const override {}
4891};
4892} // namespace
4893
4894/// -------------------- Dereferenceable Argument Attribute --------------------
4895
4896namespace {
4897struct AADereferenceableImpl : AADereferenceable {
4898 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4899 : AADereferenceable(IRP, A) {}
4900 using StateType = DerefState;
4901
4902 /// See AbstractAttribute::initialize(...).
4903 void initialize(Attributor &A) override {
4904 Value &V = *getAssociatedValue().stripPointerCasts();
4906 A.getAttrs(getIRPosition(),
4907 {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4908 Attrs, /* IgnoreSubsumingPositions */ false);
4909 for (const Attribute &Attr : Attrs)
4910 takeKnownDerefBytesMaximum(Attr.getValueAsInt());
4911
4912 // Ensure we initialize the non-null AA (if necessary).
4913 bool IsKnownNonNull;
4914 AA::hasAssumedIRAttr<Attribute::NonNull>(
4915 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
4916
4917 bool CanBeNull, CanBeFreed;
4918 takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(
4919 A.getDataLayout(), CanBeNull, CanBeFreed));
4920
4921 if (Instruction *CtxI = getCtxI())
4922 followUsesInMBEC(*this, A, getState(), *CtxI);
4923 }
4924
4925 /// See AbstractAttribute::getState()
4926 /// {
4927 StateType &getState() override { return *this; }
4928 const StateType &getState() const override { return *this; }
4929 /// }
4930
4931 /// Helper function for collecting accessed bytes in must-be-executed-context
4932 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4933 DerefState &State) {
4934 const Value *UseV = U->get();
4935 if (!UseV->getType()->isPointerTy())
4936 return;
4937
4938 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
4939 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4940 return;
4941
4942 int64_t Offset;
4944 Loc->Ptr, Offset, A.getDataLayout(), /*AllowNonInbounds*/ true);
4945 if (Base && Base == &getAssociatedValue())
4946 State.addAccessedBytes(Offset, Loc->Size.getValue());
4947 }
4948
4949 /// See followUsesInMBEC
4950 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4952 bool IsNonNull = false;
4953 bool TrackUse = false;
4954 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4955 A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);
4956 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4957 << " for instruction " << *I << "\n");
4958
4959 addAccessedBytesForUse(A, U, I, State);
4960 State.takeKnownDerefBytesMaximum(DerefBytes);
4961 return TrackUse;
4962 }
4963
4964 /// See AbstractAttribute::manifest(...).
4965 ChangeStatus manifest(Attributor &A) override {
4966 ChangeStatus Change = AADereferenceable::manifest(A);
4967 bool IsKnownNonNull;
4968 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4969 A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4970 if (IsAssumedNonNull &&
4971 A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
4972 A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
4973 return ChangeStatus::CHANGED;
4974 }
4975 return Change;
4976 }
4977
4978 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
4979 SmallVectorImpl<Attribute> &Attrs) const override {
4980 // TODO: Add *_globally support
4981 bool IsKnownNonNull;
4982 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4983 A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4984 if (IsAssumedNonNull)
4986 Ctx, getAssumedDereferenceableBytes()));
4987 else
4989 Ctx, getAssumedDereferenceableBytes()));
4990 }
4991
4992 /// See AbstractAttribute::getAsStr().
4993 const std::string getAsStr(Attributor *A) const override {
4994 if (!getAssumedDereferenceableBytes())
4995 return "unknown-dereferenceable";
4996 bool IsKnownNonNull;
4997 bool IsAssumedNonNull = false;
4998 if (A)
4999 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5000 *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
5001 return std::string("dereferenceable") +
5002 (IsAssumedNonNull ? "" : "_or_null") +
5003 (isAssumedGlobal() ? "_globally" : "") + "<" +
5004 std::to_string(getKnownDereferenceableBytes()) + "-" +
5005 std::to_string(getAssumedDereferenceableBytes()) + ">" +
5006 (!A ? " [non-null is unknown]" : "");
5007 }
5008};
5009
5010/// Dereferenceable attribute for a floating value.
5011struct AADereferenceableFloating : AADereferenceableImpl {
5012 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5013 : AADereferenceableImpl(IRP, A) {}
5014
5015 /// See AbstractAttribute::updateImpl(...).
5016 ChangeStatus updateImpl(Attributor &A) override {
5017 bool Stripped;
5018 bool UsedAssumedInformation = false;
5020 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
5021 AA::AnyScope, UsedAssumedInformation)) {
5022 Values.push_back({getAssociatedValue(), getCtxI()});
5023 Stripped = false;
5024 } else {
5025 Stripped = Values.size() != 1 ||
5026 Values.front().getValue() != &getAssociatedValue();
5027 }
5028
5029 const DataLayout &DL = A.getDataLayout();
5030 DerefState T;
5031
5032 auto VisitValueCB = [&](const Value &V) -> bool {
5033 unsigned IdxWidth =
5034 DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());
5035 APInt Offset(IdxWidth, 0);
5037 A, *this, &V, DL, Offset, /* GetMinOffset */ false,
5038 /* AllowNonInbounds */ true);
5039
5040 const auto *AA = A.getAAFor<AADereferenceable>(
5041 *this, IRPosition::value(*Base), DepClassTy::REQUIRED);
5042 int64_t DerefBytes = 0;
5043 if (!AA || (!Stripped && this == AA)) {
5044 // Use IR information if we did not strip anything.
5045 // TODO: track globally.
5046 bool CanBeNull, CanBeFreed;
5047 DerefBytes =
5048 Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
5049 T.GlobalState.indicatePessimisticFixpoint();
5050 } else {
5051 const DerefState &DS = AA->getState();
5052 DerefBytes = DS.DerefBytesState.getAssumed();
5053 T.GlobalState &= DS.GlobalState;
5054 }
5055
5056 // For now we do not try to "increase" dereferenceability due to negative
5057 // indices as we first have to come up with code to deal with loops and
5058 // for overflows of the dereferenceable bytes.
5059 int64_t OffsetSExt = Offset.getSExtValue();
5060 if (OffsetSExt < 0)
5061 OffsetSExt = 0;
5062
5063 T.takeAssumedDerefBytesMinimum(
5064 std::max(int64_t(0), DerefBytes - OffsetSExt));
5065
5066 if (this == AA) {
5067 if (!Stripped) {
5068 // If nothing was stripped IR information is all we got.
5069 T.takeKnownDerefBytesMaximum(
5070 std::max(int64_t(0), DerefBytes - OffsetSExt));
5071 T.indicatePessimisticFixpoint();
5072 } else if (OffsetSExt > 0) {
5073 // If something was stripped but there is circular reasoning we look
5074 // for the offset. If it is positive we basically decrease the
5075 // dereferenceable bytes in a circular loop now, which will simply
5076 // drive them down to the known value in a very slow way which we
5077 // can accelerate.
5078 T.indicatePessimisticFixpoint();
5079 }
5080 }
5081
5082 return T.isValidState();
5083 };
5084
5085 for (const auto &VAC : Values)
5086 if (!VisitValueCB(*VAC.getValue()))
5087 return indicatePessimisticFixpoint();
5088
5089 return clampStateAndIndicateChange(getState(), T);
5090 }
5091
5092 /// See AbstractAttribute::trackStatistics()
5093 void trackStatistics() const override {
5094 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5095 }
5096};
5097
5098/// Dereferenceable attribute for a return value.
5099struct AADereferenceableReturned final
5100 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5101 using Base =
5102 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5103 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5104 : Base(IRP, A) {}
5105
5106 /// See AbstractAttribute::trackStatistics()
5107 void trackStatistics() const override {
5108 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5109 }
5110};
5111
5112/// Dereferenceable attribute for an argument
5113struct AADereferenceableArgument final
5114 : AAArgumentFromCallSiteArguments<AADereferenceable,
5115 AADereferenceableImpl> {
5116 using Base =
5117 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5118 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5119 : Base(IRP, A) {}
5120
5121 /// See AbstractAttribute::trackStatistics()
5122 void trackStatistics() const override {
5123 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5124 }
5125};
5126
5127/// Dereferenceable attribute for a call site argument.
5128struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5129 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5130 : AADereferenceableFloating(IRP, A) {}
5131
5132 /// See AbstractAttribute::trackStatistics()
5133 void trackStatistics() const override {
5134 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5135 }
5136};
5137
5138/// Dereferenceable attribute deduction for a call site return value.
5139struct AADereferenceableCallSiteReturned final
5140 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5141 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5142 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5143 : Base(IRP, A) {}
5144
5145 /// See AbstractAttribute::trackStatistics()
5146 void trackStatistics() const override {
5147 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5148 }
5149};
5150} // namespace
5151
5152// ------------------------ Align Argument Attribute ------------------------
5153
5154namespace {
5155static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5156 Value &AssociatedValue, const Use *U,
5157 const Instruction *I, bool &TrackUse) {
5158 // We need to follow common pointer manipulation uses to the accesses they
5159 // feed into.
5160 if (isa<CastInst>(I)) {
5161 // Follow all but ptr2int casts.
5162 TrackUse = !isa<PtrToIntInst>(I);
5163 return 0;
5164 }
5165 if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
5166 if (GEP->hasAllConstantIndices())
5167 TrackUse = true;
5168 return 0;
5169 }
5170
5171 MaybeAlign MA;
5172 if (const auto *CB = dyn_cast<CallBase>(I)) {
5173 if (CB->isBundleOperand(U) || CB->isCallee(U))
5174 return 0;
5175
5176 unsigned ArgNo = CB->getArgOperandNo(U);
5177 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
5178 // As long as we only use known information there is no need to track
5179 // dependences here.
5180 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);
5181 if (AlignAA)
5182 MA = MaybeAlign(AlignAA->getKnownAlign());
5183 }
5184
5185 const DataLayout &DL = A.getDataLayout();
5186 const Value *UseV = U->get();
5187 if (auto *SI = dyn_cast<StoreInst>(I)) {
5188 if (SI->getPointerOperand() == UseV)
5189 MA = SI->getAlign();
5190 } else if (auto *LI = dyn_cast<LoadInst>(I)) {
5191 if (LI->getPointerOperand() == UseV)
5192 MA = LI->getAlign();
5193 }
5194
5195 if (!MA || *MA <= QueryingAA.getKnownAlign())
5196 return 0;
5197
5198 unsigned Alignment = MA->value();
5199 int64_t Offset;
5200
5201 if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {
5202 if (Base == &AssociatedValue) {
5203 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5204 // So we can say that the maximum power of two which is a divisor of
5205 // gcd(Offset, Alignment) is an alignment.
5206
5207 uint32_t gcd = std::gcd(uint32_t(abs((int32_t)Offset)), Alignment);
5208 Alignment = llvm::bit_floor(gcd);
5209 }
5210 }
5211
5212 return Alignment;
5213}
5214
5215struct AAAlignImpl : AAAlign {
5216 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5217
5218 /// See AbstractAttribute::initialize(...).
5219 void initialize(Attributor &A) override {
5221 A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);
5222 for (const Attribute &Attr : Attrs)
5223 takeKnownMaximum(Attr.getValueAsInt());
5224
5225 Value &V = *getAssociatedValue().stripPointerCasts();
5226 takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value());
5227
5228 if (Instruction *CtxI = getCtxI())
5229 followUsesInMBEC(*this, A, getState(), *CtxI);
5230 }
5231
5232 /// See AbstractAttribute::manifest(...).
5233 ChangeStatus manifest(Attributor &A) override {
5234 ChangeStatus LoadStoreChanged = ChangeStatus::UNCHANGED;
5235
5236 // Check for users that allow alignment annotations.
5237 Value &AssociatedValue = getAssociatedValue();
5238 for (const Use &U : AssociatedValue.uses()) {
5239 if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {
5240 if (SI->getPointerOperand() == &AssociatedValue)
5241 if (SI->getAlign() < getAssumedAlign()) {
5242 STATS_DECLTRACK(AAAlign, Store,
5243 "Number of times alignment added to a store");
5244 SI->setAlignment(getAssumedAlign());
5245 LoadStoreChanged = ChangeStatus::CHANGED;
5246 }
5247 } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
5248 if (LI->getPointerOperand() == &AssociatedValue)
5249 if (LI->getAlign() < getAssumedAlign()) {
5250 LI->setAlignment(getAssumedAlign());
5252 "Number of times alignment added to a load");
5253 LoadStoreChanged = ChangeStatus::CHANGED;
5254 }
5255 }
5256 }
5257
5258 ChangeStatus Changed = AAAlign::manifest(A);
5259
5260 Align InheritAlign =
5261 getAssociatedValue().getPointerAlignment(A.getDataLayout());
5262 if (InheritAlign >= getAssumedAlign())
5263 return LoadStoreChanged;
5264 return Changed | LoadStoreChanged;
5265 }
5266
5267 // TODO: Provide a helper to determine the implied ABI alignment and check in
5268 // the existing manifest method and a new one for AAAlignImpl that value
5269 // to avoid making the alignment explicit if it did not improve.
5270
5271 /// See AbstractAttribute::getDeducedAttributes
5272 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5273 SmallVectorImpl<Attribute> &Attrs) const override {
5274 if (getAssumedAlign() > 1)
5275 Attrs.emplace_back(
5276 Attribute::getWithAlignment(Ctx, Align(getAssumedAlign())));
5277 }
5278
5279 /// See followUsesInMBEC
5280 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5281 AAAlign::StateType &State) {
5282 bool TrackUse = false;
5283
5284 unsigned int KnownAlign =
5285 getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);
5286 State.takeKnownMaximum(KnownAlign);
5287
5288 return TrackUse;
5289 }
5290
5291 /// See AbstractAttribute::getAsStr().
5292 const std::string getAsStr(Attributor *A) const override {
5293 return "align<" + std::to_string(getKnownAlign().value()) + "-" +
5294 std::to_string(getAssumedAlign().value()) + ">";
5295 }
5296};
5297
5298/// Align attribute for a floating value.
5299struct AAAlignFloating : AAAlignImpl {
5300 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5301
5302 /// See AbstractAttribute::updateImpl(...).
5303 ChangeStatus updateImpl(Attributor &A) override {
5304 const DataLayout &DL = A.getDataLayout();
5305
5306 bool Stripped;
5307 bool UsedAssumedInformation = false;
5309 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
5310 AA::AnyScope, UsedAssumedInformation)) {
5311 Values.push_back({getAssociatedValue(), getCtxI()});
5312 Stripped = false;
5313 } else {
5314 Stripped = Values.size() != 1 ||
5315 Values.front().getValue() != &getAssociatedValue();
5316 }
5317
5318 StateType T;
5319 auto VisitValueCB = [&](Value &V) -> bool {
5320 if (isa<UndefValue>(V) || isa<ConstantPointerNull>(V))
5321 return true;
5322 const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),
5323 DepClassTy::REQUIRED);
5324 if (!AA || (!Stripped && this == AA)) {
5325 int64_t Offset;
5326 unsigned Alignment = 1;
5327 if (const Value *Base =
5329 // TODO: Use AAAlign for the base too.
5330 Align PA = Base->getPointerAlignment(DL);
5331 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5332 // So we can say that the maximum power of two which is a divisor of
5333 // gcd(Offset, Alignment) is an alignment.
5334
5335 uint32_t gcd =
5336 std::gcd(uint32_t(abs((int32_t)Offset)), uint32_t(PA.value()));
5337 Alignment = llvm::bit_floor(gcd);
5338 } else {
5339 Alignment = V.getPointerAlignment(DL).value();
5340 }
5341 // Use only IR information if we did not strip anything.
5342 T.takeKnownMaximum(Alignment);
5343 T.indicatePessimisticFixpoint();
5344 } else {
5345 // Use abstract attribute information.
5346 const AAAlign::StateType &DS = AA->getState();
5347 T ^= DS;
5348 }
5349 return T.isValidState();
5350 };
5351
5352 for (const auto &VAC : Values) {
5353 if (!VisitValueCB(*VAC.getValue()))
5354 return indicatePessimisticFixpoint();
5355 }
5356
5357 // TODO: If we know we visited all incoming values, thus no are assumed
5358 // dead, we can take the known information from the state T.
5359 return clampStateAndIndicateChange(getState(), T);
5360 }
5361
5362 /// See AbstractAttribute::trackStatistics()
5363 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5364};
5365
5366/// Align attribute for function return value.
5367struct AAAlignReturned final
5368 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5369 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5370 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5371
5372 /// See AbstractAttribute::trackStatistics()
5373 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5374};
5375
5376/// Align attribute for function argument.
5377struct AAAlignArgument final
5378 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5379 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5380 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5381
5382 /// See AbstractAttribute::manifest(...).
5383 ChangeStatus manifest(Attributor &A) override {
5384 // If the associated argument is involved in a must-tail call we give up
5385 // because we would need to keep the argument alignments of caller and
5386 // callee in-sync. Just does not seem worth the trouble right now.
5387 if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
5388 return ChangeStatus::UNCHANGED;
5389 return Base::manifest(A);
5390 }
5391
5392 /// See AbstractAttribute::trackStatistics()
5393 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5394};
5395
5396struct AAAlignCallSiteArgument final : AAAlignFloating {
5397 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5398 : AAAlignFloating(IRP, A) {}
5399
5400 /// See AbstractAttribute::manifest(...).
5401 ChangeStatus manifest(Attributor &A) override {
5402 // If the associated argument is involved in a must-tail call we give up
5403 // because we would need to keep the argument alignments of caller and
5404 // callee in-sync. Just does not seem worth the trouble right now.
5405 if (Argument *Arg = getAssociatedArgument())
5406 if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))
5407 return ChangeStatus::UNCHANGED;
5408 ChangeStatus Changed = AAAlignImpl::manifest(A);
5409 Align InheritAlign =
5410 getAssociatedValue().getPointerAlignment(A.getDataLayout());
5411 if (InheritAlign >= getAssumedAlign())
5412 Changed = ChangeStatus::UNCHANGED;
5413 return Changed;
5414 }
5415
5416 /// See AbstractAttribute::updateImpl(Attributor &A).
5417 ChangeStatus updateImpl(Attributor &A) override {
5418 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5419 if (Argument *Arg = getAssociatedArgument()) {
5420 // We only take known information from the argument
5421 // so we do not need to track a dependence.
5422 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5423 *this, IRPosition::argument(*Arg), DepClassTy::NONE);
5424 if (ArgAlignAA)
5425 takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
5426 }
5427 return Changed;
5428 }
5429
5430 /// See AbstractAttribute::trackStatistics()
5431 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5432};
5433
5434/// Align attribute deduction for a call site return value.
5435struct AAAlignCallSiteReturned final
5436 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5437 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5438 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5439 : Base(IRP, A) {}
5440
5441 /// See AbstractAttribute::trackStatistics()
5442 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5443};
5444} // namespace
5445
5446/// ------------------ Function No-Return Attribute ----------------------------
5447namespace {
5448struct AANoReturnImpl : public AANoReturn {
5449 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5450
5451 /// See AbstractAttribute::initialize(...).
5452 void initialize(Attributor &A) override {
5453 bool IsKnown;
5454 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5455 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5456 (void)IsKnown;
5457 }
5458
5459 /// See AbstractAttribute::getAsStr().
5460 const std::string getAsStr(Attributor *A) const override {
5461 return getAssumed() ? "noreturn" : "may-return";
5462 }
5463
5464 /// See AbstractAttribute::updateImpl(Attributor &A).
5465 ChangeStatus updateImpl(Attributor &A) override {
5466 auto CheckForNoReturn = [](Instruction &) { return false; };
5467 bool UsedAssumedInformation = false;
5468 if (!A.checkForAllInstructions(CheckForNoReturn, *this,
5469 {(unsigned)Instruction::Ret},
5470 UsedAssumedInformation))
5471 return indicatePessimisticFixpoint();
5472 return ChangeStatus::UNCHANGED;
5473 }
5474};
5475
5476struct AANoReturnFunction final : AANoReturnImpl {
5477 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5478 : AANoReturnImpl(IRP, A) {}
5479
5480 /// See AbstractAttribute::trackStatistics()
5481 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5482};
5483
5484/// NoReturn attribute deduction for a call sites.
5485struct AANoReturnCallSite final
5486 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5487 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5488 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5489
5490 /// See AbstractAttribute::trackStatistics()
5491 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5492};
5493} // namespace
5494
5495/// ----------------------- Instance Info ---------------------------------
5496
5497namespace {
5498/// A class to hold the state of for no-capture attributes.
5499struct AAInstanceInfoImpl : public AAInstanceInfo {
5500 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5501 : AAInstanceInfo(IRP, A) {}
5502
5503 /// See AbstractAttribute::initialize(...).
5504 void initialize(Attributor &A) override {
5505 Value &V = getAssociatedValue();
5506 if (auto *C = dyn_cast<Constant>(&V)) {
5507 if (C->isThreadDependent())
5508 indicatePessimisticFixpoint();
5509 else
5510 indicateOptimisticFixpoint();
5511 return;
5512 }
5513 if (auto *CB = dyn_cast<CallBase>(&V))
5514 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5515 !CB->mayReadFromMemory()) {
5516 indicateOptimisticFixpoint();
5517 return;
5518 }
5519 if (auto *I = dyn_cast<Instruction>(&V)) {
5520 const auto *CI =
5521 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5522 *I->getFunction());
5523 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5524 indicatePessimisticFixpoint();
5525 return;
5526 }
5527 }
5528 }
5529
5530 /// See AbstractAttribute::updateImpl(...).
5531 ChangeStatus updateImpl(Attributor &A) override {
5532 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5533
5534 Value &V = getAssociatedValue();
5535 const Function *Scope = nullptr;
5536 if (auto *I = dyn_cast<Instruction>(&V))
5537 Scope = I->getFunction();
5538 if (auto *A = dyn_cast<Argument>(&V)) {
5539 Scope = A->getParent();
5540 if (!Scope->hasLocalLinkage())
5541 return Changed;
5542 }
5543 if (!Scope)
5544 return indicateOptimisticFixpoint();
5545
5546 bool IsKnownNoRecurse;
5547 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5548 A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,
5549 IsKnownNoRecurse))
5550 return Changed;
5551
5552 auto UsePred = [&](const Use &U, bool &Follow) {
5553 const Instruction *UserI = dyn_cast<Instruction>(U.getUser());
5554 if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) ||
5555 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
5556 Follow = true;
5557 return true;
5558 }
5559 if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) ||
5560 (isa<StoreInst>(UserI) &&
5561 cast<StoreInst>(UserI)->getValueOperand() != U.get()))
5562 return true;
5563 if (auto *CB = dyn_cast<CallBase>(UserI)) {
5564 // This check is not guaranteeing uniqueness but for now that we cannot
5565 // end up with two versions of \p U thinking it was one.
5566 auto *Callee = dyn_cast_if_present<Function>(CB->getCalledOperand());
5567 if (!Callee || !Callee->hasLocalLinkage())
5568 return true;
5569 if (!CB->isArgOperand(&U))
5570 return false;
5571 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5573 DepClassTy::OPTIONAL);
5574 if (!ArgInstanceInfoAA ||
5575 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5576 return false;
5577 // If this call base might reach the scope again we might forward the
5578 // argument back here. This is very conservative.
5580 A, *CB, *Scope, *this, /* ExclusionSet */ nullptr,
5581 [Scope](const Function &Fn) { return &Fn != Scope; }))
5582 return false;
5583 return true;
5584 }
5585 return false;
5586 };
5587
5588 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5589 if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) {
5590 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5591 if ((isa<AllocaInst>(Ptr) || isNoAliasCall(Ptr)) &&
5592 AA::isDynamicallyUnique(A, *this, *Ptr))
5593 return true;
5594 }
5595 return false;
5596 };
5597
5598 if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true,
5599 DepClassTy::OPTIONAL,
5600 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5601 return indicatePessimisticFixpoint();
5602
5603 return Changed;
5604 }
5605
5606 /// See AbstractState::getAsStr().
5607 const std::string getAsStr(Attributor *A) const override {
5608 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5609 }
5610
5611 /// See AbstractAttribute::trackStatistics()
5612 void trackStatistics() const override {}
5613};
5614
5615/// InstanceInfo attribute for floating values.
5616struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5617 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5618 : AAInstanceInfoImpl(IRP, A) {}
5619};
5620
5621/// NoCapture attribute for function arguments.
5622struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5623 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5624 : AAInstanceInfoFloating(IRP, A) {}
5625};
5626
5627/// InstanceInfo attribute for call site arguments.
5628struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5629 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5630 : AAInstanceInfoImpl(IRP, A) {}
5631
5632 /// See AbstractAttribute::updateImpl(...).
5633 ChangeStatus updateImpl(Attributor &A) override {
5634 // TODO: Once we have call site specific value information we can provide
5635 // call site specific liveness information and then it makes
5636 // sense to specialize attributes for call sites arguments instead of
5637 // redirecting requests to the callee argument.
5638 Argument *Arg = getAssociatedArgument();
5639 if (!Arg)
5640 return indicatePessimisticFixpoint();
5641 const IRPosition &ArgPos = IRPosition::argument(*Arg);
5642 auto *ArgAA =
5643 A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED);
5644 if (!ArgAA)
5645 return indicatePessimisticFixpoint();
5646 return clampStateAndIndicateChange(getState(), ArgAA->getState());
5647 }
5648};
5649
5650/// InstanceInfo attribute for function return value.
5651struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5652 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5653 : AAInstanceInfoImpl(IRP, A) {
5654 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5655 }
5656
5657 /// See AbstractAttribute::initialize(...).
5658 void initialize(Attributor &A) override {
5659 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5660 }
5661
5662 /// See AbstractAttribute::updateImpl(...).
5663 ChangeStatus updateImpl(Attributor &A) override {
5664 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5665 }
5666};
5667
5668/// InstanceInfo attribute deduction for a call site return value.
5669struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5670 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5671 : AAInstanceInfoFloating(IRP, A) {}
5672};
5673} // namespace
5674
5675/// ----------------------- Variable Capturing ---------------------------------
5677 Attribute::AttrKind ImpliedAttributeKind,
5678 bool IgnoreSubsumingPositions) {
5679 assert(ImpliedAttributeKind == Attribute::NoCapture &&
5680 "Unexpected attribute kind");
5681 Value &V = IRP.getAssociatedValue();
5682 if (!IRP.isArgumentPosition())
5683 return V.use_empty();
5684
5685 // You cannot "capture" null in the default address space.
5686 if (isa<UndefValue>(V) || (isa<ConstantPointerNull>(V) &&
5687 V.getType()->getPointerAddressSpace() == 0)) {
5688 return true;
5689 }
5690
5691 if (A.hasAttr(IRP, {Attribute::NoCapture},
5692 /* IgnoreSubsumingPositions */ true, Attribute::NoCapture))
5693 return true;
5694
5695 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5696 if (Argument *Arg = IRP.getAssociatedArgument())
5697 if (A.hasAttr(IRPosition::argument(*Arg),
5698 {Attribute::NoCapture, Attribute::ByVal},
5699 /* IgnoreSubsumingPositions */ true)) {
5700 A.manifestAttrs(IRP,
5701 Attribute::get(V.getContext(), Attribute::NoCapture));
5702 return true;
5703 }
5704
5705 if (const Function *F = IRP.getAssociatedFunction()) {
5706 // Check what state the associated function can actually capture.
5708 determineFunctionCaptureCapabilities(IRP, *F, State);
5709 if (State.isKnown(NO_CAPTURE)) {
5710 A.manifestAttrs(IRP,
5711 Attribute::get(V.getContext(), Attribute::NoCapture));
5712 return true;
5713 }
5714 }
5715
5716 return false;
5717}
5718
5719/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5720/// depending on the ability of the function associated with \p IRP to capture
5721/// state in memory and through "returning/throwing", respectively.
5723 const Function &F,
5724 BitIntegerState &State) {
5725 // TODO: Once we have memory behavior attributes we should use them here.
5726
5727 // If we know we cannot communicate or write to memory, we do not care about
5728 // ptr2int anymore.
5729 bool ReadOnly = F.onlyReadsMemory();
5730 bool NoThrow = F.doesNotThrow();
5731 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5732 if (ReadOnly && NoThrow && IsVoidReturn) {
5733 State.addKnownBits(NO_CAPTURE);
5734 return;
5735 }
5736
5737 // A function cannot capture state in memory if it only reads memory, it can
5738 // however return/throw state and the state might be influenced by the
5739 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5740 if (ReadOnly)
5741 State.addKnownBits(NOT_CAPTURED_IN_MEM);
5742
5743 // A function cannot communicate state back if it does not through
5744 // exceptions and doesn not return values.
5745 if (NoThrow && IsVoidReturn)
5746 State.addKnownBits(NOT_CAPTURED_IN_RET);
5747
5748 // Check existing "returned" attributes.
5749 int ArgNo = IRP.getCalleeArgNo();
5750 if (!NoThrow || ArgNo < 0 ||
5751 !F.getAttributes().hasAttrSomewhere(Attribute::Returned))
5752 return;
5753
5754 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5755 if (F.hasParamAttribute(U, Attribute::Returned)) {
5756 if (U == unsigned(ArgNo))
5757 State.removeAssumedBits(NOT_CAPTURED_IN_RET);
5758 else if (ReadOnly)
5759 State.addKnownBits(NO_CAPTURE);
5760 else
5761 State.addKnownBits(NOT_CAPTURED_IN_RET);
5762 break;
5763 }
5764}
5765
5766namespace {
5767/// A class to hold the state of for no-capture attributes.
5768struct AANoCaptureImpl : public AANoCapture {
5769 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5770
5771 /// See AbstractAttribute::initialize(...).
5772 void initialize(Attributor &A) override {
5773 bool IsKnown;
5774 assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>(
5775 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5776 (void)IsKnown;
5777 }
5778
5779 /// See AbstractAttribute::updateImpl(...).
5780 ChangeStatus updateImpl(Attributor &A) override;
5781
5782 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5783 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5784 SmallVectorImpl<Attribute> &Attrs) const override {
5785 if (!isAssumedNoCaptureMaybeReturned())
5786 return;
5787
5788 if (isArgumentPosition()) {
5789 if (isAssumedNoCapture())
5790 Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture));
5791 else if (ManifestInternal)
5792 Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));
5793 }
5794 }
5795
5796 /// See AbstractState::getAsStr().
5797 const std::string getAsStr(Attributor *A) const override {
5798 if (isKnownNoCapture())
5799 return "known not-captured";
5800 if (isAssumedNoCapture())
5801 return "assumed not-captured";
5802 if (isKnownNoCaptureMaybeReturned())
5803 return "known not-captured-maybe-returned";
5804 if (isAssumedNoCaptureMaybeReturned())
5805 return "assumed not-captured-maybe-returned";
5806 return "assumed-captured";
5807 }
5808
5809 /// Check the use \p U and update \p State accordingly. Return true if we
5810 /// should continue to update the state.
5811 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5812 bool &Follow) {
5813 Instruction *UInst = cast<Instruction>(U.getUser());
5814 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5815 << *UInst << "\n");
5816
5817 // Deal with ptr2int by following uses.
5818 if (isa<PtrToIntInst>(UInst)) {
5819 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5820 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5821 /* Return */ true);
5822 }
5823
5824 // For stores we already checked if we can follow them, if they make it
5825 // here we give up.
5826 if (isa<StoreInst>(UInst))
5827 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5828 /* Return */ true);
5829
5830 // Explicitly catch return instructions.
5831 if (isa<ReturnInst>(UInst)) {
5832 if (UInst->getFunction() == getAnchorScope())
5833 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5834 /* Return */ true);
5835 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5836 /* Return */ true);
5837 }
5838
5839 // For now we only use special logic for call sites. However, the tracker
5840 // itself knows about a lot of other non-capturing cases already.
5841 auto *CB = dyn_cast<CallBase>(UInst);
5842 if (!CB || !CB->isArgOperand(&U))
5843 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5844 /* Return */ true);
5845
5846 unsigned ArgNo = CB->getArgOperandNo(&U);
5847 const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo);
5848 // If we have a abstract no-capture attribute for the argument we can use
5849 // it to justify a non-capture attribute here. This allows recursion!
5850 bool IsKnownNoCapture;
5851 const AANoCapture *ArgNoCaptureAA = nullptr;
5852 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
5853 A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
5854 &ArgNoCaptureAA);
5855 if (IsAssumedNoCapture)
5856 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5857 /* Return */ false);
5858 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
5859 Follow = true;
5860 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5861 /* Return */ false);
5862 }
5863
5864 // Lastly, we could not find a reason no-capture can be assumed so we don't.
5865 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5866 /* Return */ true);
5867 }
5868
5869 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
5870 /// \p CapturedInRet, then return true if we should continue updating the
5871 /// state.
5872 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
5873 bool CapturedInInt, bool CapturedInRet) {
5874 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
5875 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
5876 if (CapturedInMem)
5877 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);
5878 if (CapturedInInt)
5879 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);
5880 if (CapturedInRet)
5881 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);
5882 return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
5883 }
5884};
5885
5886ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
5887 const IRPosition &IRP = getIRPosition();
5888 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
5889 : &IRP.getAssociatedValue();
5890 if (!V)
5891 return indicatePessimisticFixpoint();
5892
5893 const Function *F =
5894 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
5895 assert(F && "Expected a function!");
5896 const IRPosition &FnPos = IRPosition::function(*F);
5897
5899
5900 // Readonly means we cannot capture through memory.
5901 bool IsKnown;
5902 if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {
5903 T.addKnownBits(NOT_CAPTURED_IN_MEM);
5904 if (IsKnown)
5905 addKnownBits(NOT_CAPTURED_IN_MEM);
5906 }
5907
5908 // Make sure all returned values are different than the underlying value.
5909 // TODO: we could do this in a more sophisticated way inside
5910 // AAReturnedValues, e.g., track all values that escape through returns
5911 // directly somehow.
5912 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
5914 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*F), this, Values,
5916 UsedAssumedInformation))
5917 return false;
5918 bool SeenConstant = false;
5919 for (const AA::ValueAndContext &VAC : Values) {
5920 if (isa<Constant>(VAC.getValue())) {
5921 if (SeenConstant)
5922 return false;
5923 SeenConstant = true;
5924 } else if (!isa<Argument>(VAC.getValue()) ||
5925 VAC.getValue() == getAssociatedArgument())
5926 return false;
5927 }
5928 return true;
5929 };
5930
5931 bool IsKnownNoUnwind;
5932 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
5933 A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
5934 bool IsVoidTy = F->getReturnType()->isVoidTy();
5935 bool UsedAssumedInformation = false;
5936 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
5937 T.addKnownBits(NOT_CAPTURED_IN_RET);
5938 if (T.isKnown(NOT_CAPTURED_IN_MEM))
5940 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
5941 addKnownBits(NOT_CAPTURED_IN_RET);
5942 if (isKnown(NOT_CAPTURED_IN_MEM))
5943 return indicateOptimisticFixpoint();
5944 }
5945 }
5946 }
5947
5948 auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
5949 const auto *DerefAA = A.getAAFor<AADereferenceable>(
5951 return DerefAA && DerefAA->getAssumedDereferenceableBytes();
5952 };
5953
5954 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
5955 switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
5957 return true;
5959 return checkUse(A, T, U, Follow);
5961 Follow = true;
5962 return true;
5963 }
5964 llvm_unreachable("Unexpected use capture kind!");
5965 };
5966
5967 if (!A.checkForAllUses(UseCheck, *this, *V))
5968 return indicatePessimisticFixpoint();
5969
5970 AANoCapture::StateType &S = getState();
5971 auto Assumed = S.getAssumed();
5972 S.intersectAssumedBits(T.getAssumed());
5973 if (!isAssumedNoCaptureMaybeReturned())
5974 return indicatePessimisticFixpoint();
5975 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
5977}
5978
5979/// NoCapture attribute for function arguments.
5980struct AANoCaptureArgument final : AANoCaptureImpl {
5981 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
5982 : AANoCaptureImpl(IRP, A) {}
5983
5984 /// See AbstractAttribute::trackStatistics()
5985 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
5986};
5987
5988/// NoCapture attribute for call site arguments.
5989struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
5990 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
5991 : AANoCaptureImpl(IRP, A) {}
5992
5993 /// See AbstractAttribute::updateImpl(...).
5994 ChangeStatus updateImpl(Attributor &A) override {
5995 // TODO: Once we have call site specific value information we can provide
5996 // call site specific liveness information and then it makes
5997 // sense to specialize attributes for call sites arguments instead of
5998 // redirecting requests to the callee argument.
5999 Argument *Arg = getAssociatedArgument();
6000 if (!Arg)
6001 return indicatePessimisticFixpoint();
6002 const IRPosition &ArgPos = IRPosition::argument(*Arg);
6003 bool IsKnownNoCapture;
6004 const AANoCapture *ArgAA = nullptr;
6005 if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
6006 A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
6007 &ArgAA))
6008 return ChangeStatus::UNCHANGED;
6009 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6010 return indicatePessimisticFixpoint();
6011 return clampStateAndIndicateChange(getState(), ArgAA->getState());
6012 }
6013
6014 /// See AbstractAttribute::trackStatistics()
6015 void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nocapture)};
6016};
6017
6018/// NoCapture attribute for floating values.
6019struct AANoCaptureFloating final : AANoCaptureImpl {
6020 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6021 : AANoCaptureImpl(IRP, A) {}
6022
6023 /// See AbstractAttribute::trackStatistics()
6024 void trackStatistics() const override {
6026 }
6027};
6028
6029/// NoCapture attribute for function return value.
6030struct AANoCaptureReturned final : AANoCaptureImpl {
6031 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6032 : AANoCaptureImpl(IRP, A) {
6033 llvm_unreachable("NoCapture is not applicable to function returns!");
6034 }
6035
6036 /// See AbstractAttribute::initialize(...).
6037 void initialize(Attributor &A) override {
6038 llvm_unreachable("NoCapture is not applicable to function returns!");
6039 }
6040
6041 /// See AbstractAttribute::updateImpl(...).
6042 ChangeStatus updateImpl(Attributor &A) override {
6043 llvm_unreachable("NoCapture is not applicable to function returns!");
6044 }
6045
6046 /// See AbstractAttribute::trackStatistics()
6047 void trackStatistics() const override {}
6048};
6049
6050/// NoCapture attribute deduction for a call site return value.
6051struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6052 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6053 : AANoCaptureImpl(IRP, A) {}
6054
6055 /// See AbstractAttribute::initialize(...).
6056 void initialize(Attributor &A) override {
6057 const Function *F = getAnchorScope();
6058 // Check what state the associated function can actually capture.
6059 determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);
6060 }
6061
6062 /// See AbstractAttribute::trackStatistics()
6063 void trackStatistics() const override {
6065 }
6066};
6067} // namespace
6068
6069/// ------------------ Value Simplify Attribute ----------------------------
6070
6071bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6072 // FIXME: Add a typecast support.
6073 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6074 SimplifiedAssociatedValue, Other, Ty);
6075 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6076 return false;
6077
6078 LLVM_DEBUG({
6079 if (SimplifiedAssociatedValue)
6080 dbgs() << "[ValueSimplify] is assumed to be "
6081 << **SimplifiedAssociatedValue << "\n";
6082 else
6083 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6084 });
6085 return true;
6086}
6087
6088namespace {
6089struct AAValueSimplifyImpl : AAValueSimplify {
6090 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6091 : AAValueSimplify(IRP, A) {}
6092
6093 /// See AbstractAttribute::initialize(...).
6094 void initialize(Attributor &A) override {
6095 if (getAssociatedValue().getType()->isVoidTy())
6096 indicatePessimisticFixpoint();
6097 if (A.hasSimplificationCallback(getIRPosition()))
6098 indicatePessimisticFixpoint();
6099 }
6100
6101 /// See AbstractAttribute::getAsStr().
6102 const std::string getAsStr(Attributor *A) const override {
6103 LLVM_DEBUG({
6104 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6105 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6106 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6107 });
6108 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6109 : "not-simple";
6110 }
6111
6112 /// See AbstractAttribute::trackStatistics()
6113 void trackStatistics() const override {}
6114
6115 /// See AAValueSimplify::getAssumedSimplifiedValue()
6116 std::optional<Value *>
6117 getAssumedSimplifiedValue(Attributor &A) const override {
6118 return SimplifiedAssociatedValue;
6119 }
6120
6121 /// Ensure the return value is \p V with type \p Ty, if not possible return
6122 /// nullptr. If \p Check is true we will only verify such an operation would
6123 /// suceed and return a non-nullptr value if that is the case. No IR is
6124 /// generated or modified.
6125 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6126 bool Check) {
6127 if (auto *TypedV = AA::getWithType(V, Ty))
6128 return TypedV;
6129 if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty))
6130 return Check ? &V
6131 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6132 &V, &Ty, "", CtxI->getIterator());
6133 return nullptr;
6134 }
6135
6136 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6137 /// If \p Check is true we will only verify such an operation would suceed and
6138 /// return a non-nullptr value if that is the case. No IR is generated or
6139 /// modified.
6140 static Value *reproduceInst(Attributor &A,
6141 const AbstractAttribute &QueryingAA,
6142 Instruction &I, Type &Ty, Instruction *CtxI,
6143 bool Check, ValueToValueMapTy &VMap) {
6144 assert(CtxI && "Cannot reproduce an instruction without context!");
6145 if (Check && (I.mayReadFromMemory() ||
6146 !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr,
6147 /* TLI */ nullptr)))
6148 return nullptr;
6149 for (Value *Op : I.operands()) {
6150 Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap);
6151 if (!NewOp) {
6152 assert(Check && "Manifest of new value unexpectedly failed!");
6153 return nullptr;
6154 }
6155 if (!Check)
6156 VMap[Op] = NewOp;
6157 }
6158 if (Check)
6159 return &I;
6160
6161 Instruction *CloneI = I.clone();
6162 // TODO: Try to salvage debug information here.
6163 CloneI->setDebugLoc(DebugLoc());
6164 VMap[&I] = CloneI;
6165 CloneI->insertBefore(CtxI);
6166 RemapInstruction(CloneI, VMap);
6167 return CloneI;
6168 }
6169
6170 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6171 /// If \p Check is true we will only verify such an operation would suceed and
6172 /// return a non-nullptr value if that is the case. No IR is generated or
6173 /// modified.
6174 static Value *reproduceValue(Attributor &A,
6175 const AbstractAttribute &QueryingAA, Value &V,
6176 Type &Ty, Instruction *CtxI, bool Check,
6177 ValueToValueMapTy &VMap) {
6178 if (const auto &NewV = VMap.lookup(&V))
6179 return NewV;
6180 bool UsedAssumedInformation = false;
6181 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6182 V, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
6183 if (!SimpleV.has_value())
6184 return PoisonValue::get(&Ty);
6185 Value *EffectiveV = &V;
6186 if (*SimpleV)
6187 EffectiveV = *SimpleV;
6188 if (auto *C = dyn_cast<Constant>(EffectiveV))
6189 return C;
6190 if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI),
6191 A.getInfoCache()))
6192 return ensureType(A, *EffectiveV, Ty, CtxI, Check);
6193 if (auto *I = dyn_cast<Instruction>(EffectiveV))
6194 if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap))
6195 return ensureType(A, *NewV, Ty, CtxI, Check);
6196 return nullptr;
6197 }
6198
6199 /// Return a value we can use as replacement for the associated one, or
6200 /// nullptr if we don't have one that makes sense.
6201 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6202 Value *NewV = SimplifiedAssociatedValue
6203 ? *SimplifiedAssociatedValue
6204 : UndefValue::get(getAssociatedType());
6205 if (NewV && NewV != &getAssociatedValue()) {
6206 ValueToValueMapTy VMap;
6207 // First verify we can reprduce the value with the required type at the
6208 // context location before we actually start modifying the IR.
6209 if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
6210 /* CheckOnly */ true, VMap))
6211 return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
6212 /* CheckOnly */ false, VMap);
6213 }
6214 return nullptr;
6215 }
6216
6217 /// Helper function for querying AAValueSimplify and updating candidate.
6218 /// \param IRP The value position we are trying to unify with SimplifiedValue
6219 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6220 const IRPosition &IRP, bool Simplify = true) {
6221 bool UsedAssumedInformation = false;
6222 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6223 if (Simplify)
6224 QueryingValueSimplified = A.getAssumedSimplified(
6225 IRP, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
6226 return unionAssumed(QueryingValueSimplified);
6227 }
6228
6229 /// Returns a candidate is found or not
6230 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6231 if (!getAssociatedValue().getType()->isIntegerTy())
6232 return false;
6233
6234 // This will also pass the call base context.
6235 const auto *AA =
6236 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6237 if (!AA)
6238 return false;
6239
6240 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6241
6242 if (!COpt) {
6243 SimplifiedAssociatedValue = std::nullopt;
6244 A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
6245 return true;
6246 }
6247 if (auto *C = *COpt) {
6248 SimplifiedAssociatedValue = C;
6249 A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
6250 return true;
6251 }
6252 return false;
6253 }
6254
6255 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6256 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6257 return true;
6258 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6259 return true;
6260 return false;
6261 }
6262
6263 /// See AbstractAttribute::manifest(...).
6264 ChangeStatus manifest(Attributor &A) override {
6265 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6266 for (auto &U : getAssociatedValue().uses()) {
6267 // Check if we need to adjust the insertion point to make sure the IR is
6268 // valid.
6269 Instruction *IP = dyn_cast<Instruction>(U.getUser());
6270 if (auto *PHI = dyn_cast_or_null<PHINode>(IP))
6271 IP = PHI->getIncomingBlock(U)->getTerminator();
6272 if (auto *NewV = manifestReplacementValue(A, IP)) {
6273 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6274 << " -> " << *NewV << " :: " << *this << "\n");
6275 if (A.changeUseAfterManifest(U, *NewV))
6276 Changed = ChangeStatus::CHANGED;
6277 }
6278 }
6279
6280 return Changed | AAValueSimplify::manifest(A);
6281 }
6282
6283 /// See AbstractState::indicatePessimisticFixpoint(...).
6284 ChangeStatus indicatePessimisticFixpoint() override {
6285 SimplifiedAssociatedValue = &getAssociatedValue();
6286 return AAValueSimplify::indicatePessimisticFixpoint();
6287 }
6288};
6289
6290struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6291 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6292 : AAValueSimplifyImpl(IRP, A) {}
6293
6294 void initialize(Attributor &A) override {
6295 AAValueSimplifyImpl::initialize(A);
6296 if (A.hasAttr(getIRPosition(),
6297 {Attribute::InAlloca, Attribute::Preallocated,
6298 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6299 /* IgnoreSubsumingPositions */ true))
6300 indicatePessimisticFixpoint();
6301 }
6302
6303 /// See AbstractAttribute::updateImpl(...).
6304 ChangeStatus updateImpl(Attributor &A) override {
6305 // Byval is only replacable if it is readonly otherwise we would write into
6306 // the replaced value and not the copy that byval creates implicitly.
6307 Argument *Arg = getAssociatedArgument();
6308 if (Arg->hasByValAttr()) {
6309 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6310 // there is no race by not copying a constant byval.
6311 bool IsKnown;
6312 if (!AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
6313 return indicatePessimisticFixpoint();
6314 }
6315
6316 auto Before = SimplifiedAssociatedValue;
6317
6318 auto PredForCallSite = [&](AbstractCallSite ACS) {
6319 const IRPosition &ACSArgPos =
6320 IRPosition::callsite_argument(ACS, getCallSiteArgNo());
6321 // Check if a coresponding argument was found or if it is on not
6322 // associated (which can happen for callback calls).
6323 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6324 return false;
6325
6326 // Simplify the argument operand explicitly and check if the result is
6327 // valid in the current scope. This avoids refering to simplified values
6328 // in other functions, e.g., we don't want to say a an argument in a
6329 // static function is actually an argument in a different function.
6330 bool UsedAssumedInformation = false;
6331 std::optional<Constant *> SimpleArgOp =
6332 A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation);
6333 if (!SimpleArgOp)
6334 return true;
6335 if (!*SimpleArgOp)
6336 return false;
6337 if (!AA::isDynamicallyUnique(A, *this, **SimpleArgOp))
6338 return false;
6339 return unionAssumed(*SimpleArgOp);
6340 };
6341
6342 // Generate a answer specific to a call site context.
6343 bool Success;
6344 bool UsedAssumedInformation = false;
6345 if (hasCallBaseContext() &&
6346 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6347 Success = PredForCallSite(
6348 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6349 else
6350 Success = A.checkForAllCallSites(PredForCallSite, *this, true,
6351 UsedAssumedInformation);
6352
6353 if (!Success)
6354 if (!askSimplifiedValueForOtherAAs(A))
6355 return indicatePessimisticFixpoint();
6356
6357 // If a candidate was found in this update, return CHANGED.
6358 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6359 : ChangeStatus ::CHANGED;
6360 }
6361
6362 /// See AbstractAttribute::trackStatistics()
6363 void trackStatistics() const override {
6364 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6365 }
6366};
6367
6368struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6369 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6370 : AAValueSimplifyImpl(IRP, A) {}
6371
6372 /// See AAValueSimplify::getAssumedSimplifiedValue()
6373 std::optional<Value *>
6374 getAssumedSimplifiedValue(Attributor &A) const override {
6375 if (!isValidState())
6376 return nullptr;
6377 return SimplifiedAssociatedValue;
6378 }
6379
6380 /// See AbstractAttribute::updateImpl(...).
6381 ChangeStatus updateImpl(Attributor &A) override {
6382 auto Before = SimplifiedAssociatedValue;
6383
6384 auto ReturnInstCB = [&](Instruction &I) {
6385 auto &RI = cast<ReturnInst>(I);
6386 return checkAndUpdate(
6387 A, *this,
6388 IRPosition::value(*RI.getReturnValue(), getCallBaseContext()));
6389 };
6390
6391 bool UsedAssumedInformation = false;
6392 if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret},
6393 UsedAssumedInformation))
6394 if (!askSimplifiedValueForOtherAAs(A))
6395 return indicatePessimisticFixpoint();
6396
6397 // If a candidate was found in this update, return CHANGED.
6398 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6399 : ChangeStatus ::CHANGED;
6400 }
6401
6402 ChangeStatus manifest(Attributor &A) override {
6403 // We queried AAValueSimplify for the returned values so they will be
6404 // replaced if a simplified form was found. Nothing to do here.
6405 return ChangeStatus::UNCHANGED;
6406 }
6407
6408 /// See AbstractAttribute::trackStatistics()
6409 void trackStatistics() const override {
6410 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6411 }
6412};
6413
6414struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6415 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6416 : AAValueSimplifyImpl(IRP, A) {}
6417
6418 /// See AbstractAttribute::initialize(...).
6419 void initialize(Attributor &A) override {
6420 AAValueSimplifyImpl::initialize(A);
6421 Value &V = getAnchorValue();
6422
6423 // TODO: add other stuffs
6424 if (isa<Constant>(V))
6425 indicatePessimisticFixpoint();
6426 }
6427
6428 /// See AbstractAttribute::updateImpl(...).
6429 ChangeStatus updateImpl(Attributor &A) override {
6430 auto Before = SimplifiedAssociatedValue;
6431 if (!askSimplifiedValueForOtherAAs(A))
6432 return indicatePessimisticFixpoint();
6433
6434 // If a candidate was found in this update, return CHANGED.
6435 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6436 : ChangeStatus ::CHANGED;
6437 }
6438
6439 /// See AbstractAttribute::trackStatistics()
6440 void trackStatistics() const override {
6441 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6442 }
6443};
6444
6445struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6446 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6447 : AAValueSimplifyImpl(IRP, A) {}
6448
6449 /// See AbstractAttribute::initialize(...).
6450 void initialize(Attributor &A) override {
6451 SimplifiedAssociatedValue = nullptr;
6452 indicateOptimisticFixpoint();
6453 }
6454 /// See AbstractAttribute::initialize(...).
6455 ChangeStatus updateImpl(Attributor &A) override {
6457 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6458 }
6459 /// See AbstractAttribute::trackStatistics()
6460 void trackStatistics() const override {
6461 STATS_DECLTRACK_FN_ATTR(value_simplify)
6462 }
6463};
6464
6465struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6466 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6467 : AAValueSimplifyFunction(IRP, A) {}
6468 /// See AbstractAttribute::trackStatistics()
6469 void trackStatistics() const override {
6470 STATS_DECLTRACK_CS_ATTR(value_simplify)
6471 }
6472};
6473
6474struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6475 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6476 : AAValueSimplifyImpl(IRP, A) {}
6477
6478 void initialize(Attributor &A) override {
6479 AAValueSimplifyImpl::initialize(A);
6480 Function *Fn = getAssociatedFunction();
6481 assert(Fn && "Did expect an associted function");
6482 for (Argument &Arg : Fn->args()) {
6483 if (Arg.hasReturnedAttr()) {
6484 auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()),
6485 Arg.getArgNo());
6487 checkAndUpdate(A, *this, IRP))
6488 indicateOptimisticFixpoint();
6489 else
6490 indicatePessimisticFixpoint();
6491 return;
6492 }
6493 }
6494 }
6495
6496 /// See AbstractAttribute::updateImpl(...).
6497 ChangeStatus updateImpl(Attributor &A) override {
6498 return indicatePessimisticFixpoint();
6499 }
6500
6501 void trackStatistics() const override {
6502 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6503 }
6504};
6505
6506struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6507 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6508 : AAValueSimplifyFloating(IRP, A) {}
6509
6510 /// See AbstractAttribute::manifest(...).
6511 ChangeStatus manifest(Attributor &A) override {
6512 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6513 // TODO: We should avoid simplification duplication to begin with.
6514 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6515 IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE);
6516 if (FloatAA && FloatAA->getState().isValidState())
6517 return Changed;
6518
6519 if (auto *NewV = manifestReplacementValue(A, getCtxI())) {
6520 Use &U = cast<CallBase>(&getAnchorValue())
6521 ->getArgOperandUse(getCallSiteArgNo());
6522 if (A.changeUseAfterManifest(U, *NewV))
6523 Changed = ChangeStatus::CHANGED;
6524 }
6525
6526 return Changed | AAValueSimplify::manifest(A);
6527 }
6528
6529 void trackStatistics() const override {
6530 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6531 }
6532};
6533} // namespace
6534
6535/// ----------------------- Heap-To-Stack Conversion ---------------------------
6536namespace {
6537struct AAHeapToStackFunction final : public AAHeapToStack {
6538
6539 struct AllocationInfo {
6540 /// The call that allocates the memory.
6541 CallBase *const CB;
6542
6543 /// The library function id for the allocation.
6544 LibFunc LibraryFunctionId = NotLibFunc;
6545
6546 /// The status wrt. a rewrite.
6547 enum {
6548 STACK_DUE_TO_USE,
6549 STACK_DUE_TO_FREE,
6550 INVALID,
6551 } Status = STACK_DUE_TO_USE;
6552
6553 /// Flag to indicate if we encountered a use that might free this allocation
6554 /// but which is not in the deallocation infos.
6555 bool HasPotentiallyFreeingUnknownUses = false;
6556
6557 /// Flag to indicate that we should place the new alloca in the function
6558 /// entry block rather than where the call site (CB) is.
6559 bool MoveAllocaIntoEntry = true;
6560
6561 /// The set of free calls that use this allocation.
6562 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6563 };
6564
6565 struct DeallocationInfo {
6566 /// The call that deallocates the memory.
6567 CallBase *const CB;
6568 /// The value freed by the call.
6569 Value *FreedOp;
6570
6571 /// Flag to indicate if we don't know all objects this deallocation might
6572 /// free.
6573 bool MightFreeUnknownObjects = false;
6574
6575 /// The set of allocation calls that are potentially freed.
6576 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6577 };
6578
6579 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6580 : AAHeapToStack(IRP, A) {}
6581
6582 ~AAHeapToStackFunction() {
6583 // Ensure we call the destructor so we release any memory allocated in the
6584 // sets.
6585 for (auto &It : AllocationInfos)
6586 It.second->~AllocationInfo();
6587 for (auto &It : DeallocationInfos)
6588 It.second->~DeallocationInfo();
6589 }
6590
6591 void initialize(Attributor &A) override {
6592 AAHeapToStack::initialize(A);
6593
6594 const Function *F = getAnchorScope();
6595 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6596
6597 auto AllocationIdentifierCB = [&](Instruction &I) {
6598 CallBase *CB = dyn_cast<CallBase>(&I);
6599 if (!CB)
6600 return true;
6601 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6602 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{CB, FreedOp};
6603 return true;
6604 }
6605 // To do heap to stack, we need to know that the allocation itself is
6606 // removable once uses are rewritten, and that we can initialize the
6607 // alloca to the same pattern as the original allocation result.
6608 if (isRemovableAlloc(CB, TLI)) {
6609 auto *I8Ty = Type::getInt8Ty(CB->getParent()->getContext());
6610 if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) {
6611 AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB};
6612 AllocationInfos[CB] = AI;
6613 if (TLI)
6614 TLI->getLibFunc(*CB, AI->LibraryFunctionId);
6615 }
6616 }
6617 return true;
6618 };
6619
6620 bool UsedAssumedInformation = false;
6621 bool Success = A.checkForAllCallLikeInstructions(
6622 AllocationIdentifierCB, *this, UsedAssumedInformation,
6623 /* CheckBBLivenessOnly */ false,
6624 /* CheckPotentiallyDead */ true);
6625 (void)Success;
6626 assert(Success && "Did not expect the call base visit callback to fail!");
6627
6629 [](const IRPosition &, const AbstractAttribute *,
6630 bool &) -> std::optional<Value *> { return nullptr; };
6631 for (const auto &It : AllocationInfos)
6632 A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
6633 SCB);
6634 for (const auto &It : DeallocationInfos)
6635 A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
6636 SCB);
6637 }
6638
6639 const std::string getAsStr(Attributor *A) const override {
6640 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6641 for (const auto &It : AllocationInfos) {
6642 if (It.second->Status == AllocationInfo::INVALID)
6643 ++NumInvalidMallocs;
6644 else
6645 ++NumH2SMallocs;
6646 }
6647 return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) + "/" +
6648 std::to_string(NumInvalidMallocs);
6649 }
6650
6651 /// See AbstractAttribute::trackStatistics().
6652 void trackStatistics() const override {
6653 STATS_DECL(
6654 MallocCalls, Function,
6655 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6656 for (const auto &It : AllocationInfos)
6657 if (It.second->Status != AllocationInfo::INVALID)
6658 ++BUILD_STAT_NAME(MallocCalls, Function);
6659 }
6660
6661 bool isAssumedHeapToStack(const CallBase &CB) const override {
6662 if (isValidState())
6663 if (AllocationInfo *AI =
6664 AllocationInfos.lookup(const_cast<CallBase *>(&CB)))
6665 return AI->Status != AllocationInfo::INVALID;
6666 return false;
6667 }
6668
6669 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6670 if (!isValidState())
6671 return false;
6672
6673 for (const auto &It : AllocationInfos) {
6674 AllocationInfo &AI = *It.second;
6675 if (AI.Status == AllocationInfo::INVALID)
6676 continue;
6677
6678 if (AI.PotentialFreeCalls.count(&CB))
6679 return true;
6680 }
6681
6682 return false;
6683 }
6684
6685 ChangeStatus manifest(Attributor &A) override {
6686 assert(getState().isValidState() &&
6687 "Attempted to manifest an invalid state!");
6688
6689 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6690 Function *F = getAnchorScope();
6691 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6692
6693 for (auto &It : AllocationInfos) {
6694 AllocationInfo &AI = *It.second;
6695 if (AI.Status == AllocationInfo::INVALID)
6696 continue;
6697
6698 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6699 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6700 A.deleteAfterManifest(*FreeCall);
6701 HasChanged = ChangeStatus::CHANGED;
6702 }
6703
6704 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6705 << "\n");
6706
6707 auto Remark = [&](OptimizationRemark OR) {
6708 LibFunc IsAllocShared;
6709 if (TLI->getLibFunc(*AI.CB, IsAllocShared))
6710 if (IsAllocShared == LibFunc___kmpc_alloc_shared)
6711 return OR << "Moving globalized variable to the stack.";
6712 return OR << "Moving memory allocation from the heap to the stack.";
6713 };
6714 if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
6715 A.emitRemark<OptimizationRemark>(AI.CB, "OMP110", Remark);
6716 else
6717 A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark);
6718
6719 const DataLayout &DL = A.getInfoCache().getDL();
6720 Value *Size;
6721 std::optional<APInt> SizeAPI = getSize(A, *this, AI);
6722 if (SizeAPI) {
6723 Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
6724 } else {
6725 LLVMContext &Ctx = AI.CB->getContext();
6726 ObjectSizeOpts Opts;
6727 ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
6728 SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
6729 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6730 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6731 Size = SizeOffsetPair.Size;
6732 }
6733
6734 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6735 ? F->getEntryBlock().begin()
6736 : AI.CB->getIterator();
6737
6738 Align Alignment(1);
6739 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6740 Alignment = std::max(Alignment, *RetAlign);
6741 if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
6742 std::optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align);
6743 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6744 "Expected an alignment during manifest!");
6745 Alignment =
6746 std::max(Alignment, assumeAligned(AlignmentAPI->getZExtValue()));
6747 }
6748
6749 // TODO: Hoist the alloca towards the function entry.
6750 unsigned AS = DL.getAllocaAddrSpace();
6751 Instruction *Alloca =
6752 new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,
6753 AI.CB->getName() + ".h2s", IP);
6754
6755 if (Alloca->getType() != AI.CB->getType())
6756 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6757 Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator());
6758
6759 auto *I8Ty = Type::getInt8Ty(F->getContext());
6760 auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty);
6761 assert(InitVal &&
6762 "Must be able to materialize initial memory state of allocation");
6763
6764 A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca);
6765
6766 if (auto *II = dyn_cast<InvokeInst>(AI.CB)) {
6767 auto *NBB = II->getNormalDest();
6768 BranchInst::Create(NBB, AI.CB->getParent());
6769 A.deleteAfterManifest(*AI.CB);
6770 } else {
6771 A.deleteAfterManifest(*AI.CB);
6772 }
6773
6774 // Initialize the alloca with the same value as used by the allocation
6775 // function. We can skip undef as the initial value of an alloc is
6776 // undef, and the memset would simply end up being DSEd.
6777 if (!isa<UndefValue>(InitVal)) {
6778 IRBuilder<> Builder(Alloca->getNextNode());
6779 // TODO: Use alignment above if align!=1
6780 Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);
6781 }
6782 HasChanged = ChangeStatus::CHANGED;
6783 }
6784
6785 return HasChanged;
6786 }
6787
6788 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6789 Value &V) {
6790 bool UsedAssumedInformation = false;
6791 std::optional<Constant *> SimpleV =
6792 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6793 if (!SimpleV)
6794 return APInt(64, 0);
6795 if (auto *CI = dyn_cast_or_null<ConstantInt>(*SimpleV))
6796 return CI->getValue();
6797 return std::nullopt;
6798 }
6799
6800 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6801 AllocationInfo &AI) {
6802 auto Mapper = [&](const Value *V) -> const Value * {
6803 bool UsedAssumedInformation = false;
6804 if (std::optional<Constant *> SimpleV =
6805 A.getAssumedConstant(*V, AA, UsedAssumedInformation))
6806 if (*SimpleV)
6807 return *SimpleV;
6808 return V;
6809 };
6810
6811 const Function *F = getAnchorScope();
6812 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6813 return getAllocSize(AI.CB, TLI, Mapper);
6814 }
6815
6816 /// Collection of all malloc-like calls in a function with associated
6817 /// information.
6819
6820 /// Collection of all free-like calls in a function with associated
6821 /// information.
6823
6824 ChangeStatus updateImpl(Attributor &A) override;
6825};
6826
6827ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6829 const Function *F = getAnchorScope();
6830 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6831
6832 const auto *LivenessAA =
6833 A.getAAFor<AAIsDead>(*this, IRPosition::function(*F), DepClassTy::NONE);
6834
6836 A.getInfoCache().getMustBeExecutedContextExplorer();
6837
6838 bool StackIsAccessibleByOtherThreads =
6839 A.getInfoCache().stackIsAccessibleByOtherThreads();
6840
6841 LoopInfo *LI =
6842 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);
6843 std::optional<bool> MayContainIrreducibleControl;
6844 auto IsInLoop = [&](BasicBlock &BB) {
6845 if (&F->getEntryBlock() == &BB)
6846 return false;
6847 if (!MayContainIrreducibleControl.has_value())
6848 MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI);
6849 if (*MayContainIrreducibleControl)
6850 return true;
6851 if (!LI)
6852 return true;
6853 return LI->getLoopFor(&BB) != nullptr;
6854 };
6855
6856 // Flag to ensure we update our deallocation information at most once per
6857 // updateImpl call and only if we use the free check reasoning.
6858 bool HasUpdatedFrees = false;
6859
6860 auto UpdateFrees = [&]() {
6861 HasUpdatedFrees = true;
6862
6863 for (auto &It : DeallocationInfos) {
6864 DeallocationInfo &DI = *It.second;
6865 // For now we cannot use deallocations that have unknown inputs, skip
6866 // them.
6867 if (DI.MightFreeUnknownObjects)
6868 continue;
6869
6870 // No need to analyze dead calls, ignore them instead.
6871 bool UsedAssumedInformation = false;
6872 if (A.isAssumedDead(*DI.CB, this, LivenessAA, UsedAssumedInformation,
6873 /* CheckBBLivenessOnly */ true))
6874 continue;
6875
6876 // Use the non-optimistic version to get the freed object.
6877 Value *Obj = getUnderlyingObject(DI.FreedOp);
6878 if (!Obj) {
6879 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
6880 DI.MightFreeUnknownObjects = true;
6881 continue;
6882 }
6883
6884 // Free of null and undef can be ignored as no-ops (or UB in the latter
6885 // case).
6886 if (isa<ConstantPointerNull>(Obj) || isa<UndefValue>(Obj))
6887 continue;
6888
6889 CallBase *ObjCB = dyn_cast<CallBase>(Obj);
6890 if (!ObjCB) {
6891 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
6892 << "\n");
6893 DI.MightFreeUnknownObjects = true;
6894 continue;
6895 }
6896
6897 AllocationInfo *AI = AllocationInfos.lookup(ObjCB);
6898 if (!AI) {
6899 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
6900 << "\n");
6901 DI.MightFreeUnknownObjects = true;
6902 continue;
6903 }
6904
6905 DI.PotentialAllocationCalls.insert(ObjCB);
6906 }
6907 };
6908
6909 auto FreeCheck = [&](AllocationInfo &AI) {
6910 // If the stack is not accessible by other threads, the "must-free" logic
6911 // doesn't apply as the pointer could be shared and needs to be places in
6912 // "shareable" memory.
6913 if (!StackIsAccessibleByOtherThreads) {
6914 bool IsKnownNoSycn;
6915 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
6916 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {
6917 LLVM_DEBUG(
6918 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
6919 "other threads and function is not nosync:\n");
6920 return false;
6921 }
6922 }
6923 if (!HasUpdatedFrees)
6924 UpdateFrees();
6925
6926 // TODO: Allow multi exit functions that have different free calls.
6927 if (AI.PotentialFreeCalls.size() != 1) {
6928 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
6929 << AI.PotentialFreeCalls.size() << "\n");
6930 return false;
6931 }
6932 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
6933 DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);
6934 if (!DI) {
6935 LLVM_DEBUG(
6936 dbgs() << "[H2S] unique free call was not known as deallocation call "
6937 << *UniqueFree << "\n");
6938 return false;
6939 }
6940 if (DI->MightFreeUnknownObjects) {
6941 LLVM_DEBUG(
6942 dbgs() << "[H2S] unique free call might free unknown allocations\n");
6943 return false;
6944 }
6945 if (DI->PotentialAllocationCalls.empty())
6946 return true;
6947 if (DI->PotentialAllocationCalls.size() > 1) {
6948 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
6949 << DI->PotentialAllocationCalls.size()
6950 << " different allocations\n");
6951 return false;
6952 }
6953 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
6954 LLVM_DEBUG(
6955 dbgs()
6956 << "[H2S] unique free call not known to free this allocation but "
6957 << **DI->PotentialAllocationCalls.begin() << "\n");
6958 return false;
6959 }
6960
6961 // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
6962 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
6963 Instruction *CtxI = isa<InvokeInst>(AI.CB) ? AI.CB : AI.CB->getNextNode();
6964 if (!Explorer || !Explorer->findInContextOf(UniqueFree, CtxI)) {
6965 LLVM_DEBUG(
6966 dbgs()
6967 << "[H2S] unique free call might not be executed with the allocation "
6968 << *UniqueFree << "\n");
6969 return false;
6970 }
6971 }
6972 return true;
6973 };
6974
6975 auto UsesCheck = [&](AllocationInfo &AI) {
6976 bool ValidUsesOnly = true;
6977
6978 auto Pred = [&](const Use &U, bool &Follow) -> bool {
6979 Instruction *UserI = cast<Instruction>(U.getUser());
6980 if (isa<LoadInst>(UserI))
6981 return true;
6982 if (auto *SI = dyn_cast<StoreInst>(UserI)) {
6983 if (SI->getValueOperand() == U.get()) {
6985 << "[H2S] escaping store to memory: " << *UserI << "\n");
6986 ValidUsesOnly = false;
6987 } else {
6988 // A store into the malloc'ed memory is fine.
6989 }
6990 return true;
6991 }
6992 if (auto *CB = dyn_cast<CallBase>(UserI)) {
6993 if (!CB->isArgOperand(&U) || CB->isLifetimeStartOrEnd())
6994 return true;
6995 if (DeallocationInfos.count(CB)) {
6996 AI.PotentialFreeCalls.insert(CB);
6997 return true;
6998 }
6999
7000 unsigned ArgNo = CB->getArgOperandNo(&U);
7001 auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);
7002
7003 bool IsKnownNoCapture;
7004 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
7005 A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
7006
7007 // If a call site argument use is nofree, we are fine.
7008 bool IsKnownNoFree;
7009 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7010 A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);
7011
7012 if (!IsAssumedNoCapture ||
7013 (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7014 !IsAssumedNoFree)) {
7015 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7016
7017 // Emit a missed remark if this is missed OpenMP globalization.
7018 auto Remark = [&](OptimizationRemarkMissed ORM) {
7019 return ORM
7020 << "Could not move globalized variable to the stack. "
7021 "Variable is potentially captured in call. Mark "
7022 "parameter as `__attribute__((noescape))` to override.";
7023 };
7024
7025 if (ValidUsesOnly &&
7026 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
7027 A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark);
7028
7029 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7030 ValidUsesOnly = false;
7031 }
7032 return true;
7033 }
7034
7035 if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
7036 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
7037 Follow = true;
7038 return true;
7039 }
7040 // Unknown user for which we can not track uses further (in a way that
7041 // makes sense).
7042 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7043 ValidUsesOnly = false;
7044 return true;
7045 };
7046 if (!A.checkForAllUses(Pred, *this, *AI.CB, /* CheckBBLivenessOnly */ false,
7047 DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7048 [&](const Use &OldU, const Use &NewU) {
7049 auto *SI = dyn_cast<StoreInst>(OldU.getUser());
7050 return !SI || StackIsAccessibleByOtherThreads ||
7051 AA::isAssumedThreadLocalObject(
7052 A, *SI->getPointerOperand(), *this);
7053 }))
7054 return false;
7055 return ValidUsesOnly;
7056 };
7057
7058 // The actual update starts here. We look at all allocations and depending on
7059 // their status perform the appropriate check(s).
7060 for (auto &It : AllocationInfos) {
7061 AllocationInfo &AI = *It.second;
7062 if (AI.Status == AllocationInfo::INVALID)
7063 continue;
7064
7065 if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
7066 std::optional<APInt> APAlign = getAPInt(A, *this, *Align);
7067 if (!APAlign) {
7068 // Can't generate an alloca which respects the required alignment
7069 // on the allocation.
7070 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7071 << "\n");
7072 AI.Status = AllocationInfo::INVALID;
7073 Changed = ChangeStatus::CHANGED;
7074 continue;
7075 }
7076 if (APAlign->ugt(llvm::Value::MaximumAlignment) ||
7077 !APAlign->isPowerOf2()) {
7078 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7079 << "\n");
7080 AI.Status = AllocationInfo::INVALID;
7081 Changed = ChangeStatus::CHANGED;
7082 continue;
7083 }
7084 }
7085
7086 std::optional<APInt> Size = getSize(A, *this, AI);
7087 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7088 MaxHeapToStackSize != -1) {
7089 if (!Size || Size->ugt(MaxHeapToStackSize)) {
7090 LLVM_DEBUG({
7091 if (!Size)
7092 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7093 else
7094 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7095 << MaxHeapToStackSize << "\n";
7096 });
7097
7098 AI.Status = AllocationInfo::INVALID;
7099 Changed = ChangeStatus::CHANGED;
7100 continue;
7101 }
7102 }
7103
7104 switch (AI.Status) {
7105 case AllocationInfo::STACK_DUE_TO_USE:
7106 if (UsesCheck(AI))
7107 break;
7108 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7109 [[fallthrough]];
7110 case AllocationInfo::STACK_DUE_TO_FREE:
7111 if (FreeCheck(AI))
7112 break;
7113 AI.Status = AllocationInfo::INVALID;
7114 Changed = ChangeStatus::CHANGED;
7115 break;
7116 case AllocationInfo::INVALID:
7117 llvm_unreachable("Invalid allocations should never reach this point!");
7118 };
7119
7120 // Check if we still think we can move it into the entry block. If the
7121 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7122 // ignore the potential compilations associated with loops.
7123 bool IsGlobalizedLocal =
7124 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
7125 if (AI.MoveAllocaIntoEntry &&
7126 (!Size.has_value() ||
7127 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7128 AI.MoveAllocaIntoEntry = false;
7129 }
7130
7131 return Changed;
7132}
7133} // namespace
7134
7135/// ----------------------- Privatizable Pointers ------------------------------
7136namespace {
7137struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7138 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7139 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7140
7141 ChangeStatus indicatePessimisticFixpoint() override {
7142 AAPrivatizablePtr::indicatePessimisticFixpoint();
7143 PrivatizableType = nullptr;
7144 return ChangeStatus::CHANGED;
7145 }
7146
7147 /// Identify the type we can chose for a private copy of the underlying
7148 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7149 /// none.
7150 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7151
7152 /// Return a privatizable type that encloses both T0 and T1.
7153 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7154 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7155 std::optional<Type *> T1) {
7156 if (!T0)
7157 return T1;
7158 if (!T1)
7159 return T0;
7160 if (T0 == T1)
7161 return T0;
7162 return nullptr;
7163 }
7164
7165 std::optional<Type *> getPrivatizableType() const override {
7166 return PrivatizableType;
7167 }
7168
7169 const std::string getAsStr(Attributor *A) const override {
7170 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7171 }
7172
7173protected:
7174 std::optional<Type *> PrivatizableType;
7175};
7176
7177// TODO: Do this for call site arguments (probably also other values) as well.
7178
7179struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7180 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7181 : AAPrivatizablePtrImpl(IRP, A) {}
7182
7183 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7184 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7185 // If this is a byval argument and we know all the call sites (so we can
7186 // rewrite them), there is no need to check them explicitly.
7187 bool UsedAssumedInformation = false;
7189 A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,
7190 /* IgnoreSubsumingPositions */ true);
7191 if (!Attrs.empty() &&
7192 A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this,
7193 true, UsedAssumedInformation))
7194 return Attrs[0].getValueAsType();
7195
7196 std::optional<Type *> Ty;
7197 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7198
7199 // Make sure the associated call site argument has the same type at all call
7200 // sites and it is an allocation we know is safe to privatize, for now that
7201 // means we only allow alloca instructions.
7202 // TODO: We can additionally analyze the accesses in the callee to create
7203 // the type from that information instead. That is a little more
7204 // involved and will be done in a follow up patch.
7205 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7206 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7207 // Check if a coresponding argument was found or if it is one not
7208 // associated (which can happen for callback calls).
7209 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7210 return false;
7211
7212 // Check that all call sites agree on a type.
7213 auto *PrivCSArgAA =
7214 A.getAAFor<AAPrivatizablePtr>(*this, ACSArgPos, DepClassTy::REQUIRED);
7215 if (!PrivCSArgAA)
7216 return false;
7217 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7218
7219 LLVM_DEBUG({
7220 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7221 if (CSTy && *CSTy)
7222 (*CSTy)->print(dbgs());
7223 else if (CSTy)
7224 dbgs() << "<nullptr>";
7225 else
7226 dbgs() << "<none>";
7227 });
7228
7229 Ty = combineTypes(Ty, CSTy);
7230
7231 LLVM_DEBUG({
7232 dbgs() << " : New Type: ";
7233 if (Ty && *Ty)
7234 (*Ty)->print(dbgs());
7235 else if (Ty)
7236 dbgs() << "<nullptr>";
7237 else
7238 dbgs() << "<none>";
7239 dbgs() << "\n";
7240 });
7241
7242 return !Ty || *Ty;
7243 };
7244
7245 if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
7246 UsedAssumedInformation))
7247 return nullptr;
7248 return Ty;
7249 }
7250
7251 /// See AbstractAttribute::updateImpl(...).
7252 ChangeStatus updateImpl(Attributor &A) override {
7253 PrivatizableType = identifyPrivatizableType(A);
7254 if (!PrivatizableType)
7255 return ChangeStatus::UNCHANGED;
7256 if (!*PrivatizableType)
7257 return indicatePessimisticFixpoint();
7258
7259 // The dependence is optional so we don't give up once we give up on the
7260 // alignment.
7261 A.getAAFor<AAAlign>(*this, IRPosition::value(getAssociatedValue()),
7262 DepClassTy::OPTIONAL);
7263
7264 // Avoid arguments with padding for now.
7265 if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&
7266 !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {
7267 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7268 return indicatePessimisticFixpoint();
7269 }
7270
7271 // Collect the types that will replace the privatizable type in the function
7272 // signature.
7273 SmallVector<Type *, 16> ReplacementTypes;
7274 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7275
7276 // Verify callee and caller agree on how the promoted argument would be
7277 // passed.
7278 Function &Fn = *getIRPosition().getAnchorScope();
7279 const auto *TTI =
7280 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
7281 if (!TTI) {
7282 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7283 << Fn.getName() << "\n");
7284 return indicatePessimisticFixpoint();
7285 }
7286
7287 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7288 CallBase *CB = ACS.getInstruction();
7289 return TTI->areTypesABICompatible(
7290 CB->getCaller(),
7291 dyn_cast_if_present<Function>(CB->getCalledOperand()),
7292 ReplacementTypes);
7293 };
7294 bool UsedAssumedInformation = false;
7295 if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
7296 UsedAssumedInformation)) {
7297 LLVM_DEBUG(
7298 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7299 << Fn.getName() << "\n");
7300 return indicatePessimisticFixpoint();
7301 }
7302
7303 // Register a rewrite of the argument.
7304 Argument *Arg = getAssociatedArgument();
7305 if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
7306 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7307 return indicatePessimisticFixpoint();
7308 }
7309
7310 unsigned ArgNo = Arg->getArgNo();
7311
7312 // Helper to check if for the given call site the associated argument is
7313 // passed to a callback where the privatization would be different.
7314 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7315 SmallVector<const Use *, 4> CallbackUses;
7316 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7317 for (const Use *U : CallbackUses) {
7318 AbstractCallSite CBACS(U);
7319 assert(CBACS && CBACS.isCallbackCall());
7320 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7321 int CBArgNo = CBACS.getCallArgOperandNo(CBArg);
7322
7323 LLVM_DEBUG({
7324 dbgs()
7325 << "[AAPrivatizablePtr] Argument " << *Arg
7326 << "check if can be privatized in the context of its parent ("
7327 << Arg->getParent()->getName()
7328 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7329 "callback ("
7330 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7331 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7332 << CBACS.getCallArgOperand(CBArg) << " vs "
7333 << CB.getArgOperand(ArgNo) << "\n"
7334 << "[AAPrivatizablePtr] " << CBArg << " : "
7335 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7336 });
7337
7338 if (CBArgNo != int(ArgNo))
7339 continue;
7340 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7341 *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED);
7342 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7343 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7344 if (!CBArgPrivTy)
7345 continue;
7346 if (*CBArgPrivTy == PrivatizableType)
7347 continue;
7348 }
7349
7350 LLVM_DEBUG({
7351 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7352 << " cannot be privatized in the context of its parent ("
7353 << Arg->getParent()->getName()
7354 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7355 "callback ("
7356 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7357 << ").\n[AAPrivatizablePtr] for which the argument "
7358 "privatization is not compatible.\n";
7359 });
7360 return false;
7361 }
7362 }
7363 return true;
7364 };
7365
7366 // Helper to check if for the given call site the associated argument is
7367 // passed to a direct call where the privatization would be different.
7368 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7369 CallBase *DC = cast<CallBase>(ACS.getInstruction());
7370 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7371 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7372 "Expected a direct call operand for callback call operand");
7373
7374 Function *DCCallee =
7375 dyn_cast_if_present<Function>(DC->getCalledOperand());
7376 LLVM_DEBUG({
7377 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7378 << " check if be privatized in the context of its parent ("
7379 << Arg->getParent()->getName()
7380 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7381 "direct call of ("
7382 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7383 });
7384
7385 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7386 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7387 *this, IRPosition::argument(*DCCallee->getArg(DCArgNo)),
7388 DepClassTy::REQUIRED);
7389 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7390 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7391 if (!DCArgPrivTy)
7392 return true;
7393 if (*DCArgPrivTy == PrivatizableType)
7394 return true;
7395 }
7396 }
7397
7398 LLVM_DEBUG({
7399 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7400 << " cannot be privatized in the context of its parent ("
7401 << Arg->getParent()->getName()
7402 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7403 "direct call of ("
7405 << ").\n[AAPrivatizablePtr] for which the argument "
7406 "privatization is not compatible.\n";
7407 });
7408 return false;
7409 };
7410
7411 // Helper to check if the associated argument is used at the given abstract
7412 // call site in a way that is incompatible with the privatization assumed
7413 // here.
7414 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7415 if (ACS.isDirectCall())
7416 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7417 if (ACS.isCallbackCall())
7418 return IsCompatiblePrivArgOfDirectCS(ACS);
7419 return false;
7420 };
7421
7422 if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,
7423 UsedAssumedInformation))
7424 return indicatePessimisticFixpoint();
7425
7426 return ChangeStatus::UNCHANGED;
7427 }
7428
7429 /// Given a type to private \p PrivType, collect the constituates (which are
7430 /// used) in \p ReplacementTypes.
7431 static void
7432 identifyReplacementTypes(Type *PrivType,
7433 SmallVectorImpl<Type *> &ReplacementTypes) {
7434 // TODO: For now we expand the privatization type to the fullest which can
7435 // lead to dead arguments that need to be removed later.
7436 assert(PrivType && "Expected privatizable type!");
7437
7438 // Traverse the type, extract constituate types on the outermost level.
7439 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7440 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7441 ReplacementTypes.push_back(PrivStructType->getElementType(u));
7442 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7443 ReplacementTypes.append(PrivArrayType->getNumElements(),
7444 PrivArrayType->getElementType());
7445 } else {
7446 ReplacementTypes.push_back(PrivType);
7447 }
7448 }
7449
7450 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7451 /// The values needed are taken from the arguments of \p F starting at
7452 /// position \p ArgNo.
7453 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7454 unsigned ArgNo, BasicBlock::iterator IP) {
7455 assert(PrivType && "Expected privatizable type!");
7456
7457 IRBuilder<NoFolder> IRB(IP->getParent(), IP);
7458 const DataLayout &DL = F.getParent()->getDataLayout();
7459
7460 // Traverse the type, build GEPs and stores.
7461 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7462 const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
7463 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7464 Value *Ptr =
7465 constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB);
7466 new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
7467 }
7468 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7469 Type *PointeeTy = PrivArrayType->getElementType();
7470 uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
7471 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7472 Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB);
7473 new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
7474 }
7475 } else {
7476 new StoreInst(F.getArg(ArgNo), &Base, IP);
7477 }
7478 }
7479
7480 /// Extract values from \p Base according to the type \p PrivType at the
7481 /// call position \p ACS. The values are appended to \p ReplacementValues.
7482 void createReplacementValues(Align Alignment, Type *PrivType,
7484 SmallVectorImpl<Value *> &ReplacementValues) {
7485 assert(Base && "Expected base value!");
7486 assert(PrivType && "Expected privatizable type!");
7487 Instruction *IP = ACS.getInstruction();
7488
7489 IRBuilder<NoFolder> IRB(IP);
7490 const DataLayout &DL = IP->getModule()->getDataLayout();
7491
7492 // Traverse the type, build GEPs and loads.
7493 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7494 const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
7495 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7496 Type *PointeeTy = PrivStructType->getElementType(u);
7497 Value *Ptr =
7498 constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB);
7499 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7500 L->setAlignment(Alignment);
7501 ReplacementValues.push_back(L);
7502 }
7503 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7504 Type *PointeeTy = PrivArrayType->getElementType();
7505 uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
7506 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7507 Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB);
7508 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7509 L->setAlignment(Alignment);
7510 ReplacementValues.push_back(L);
7511 }
7512 } else {
7513 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7514 L->setAlignment(Alignment);
7515 ReplacementValues.push_back(L);
7516 }
7517 }
7518
7519 /// See AbstractAttribute::manifest(...)
7520 ChangeStatus manifest(Attributor &A) override {
7521 if (!PrivatizableType)
7522 return ChangeStatus::UNCHANGED;
7523 assert(*PrivatizableType && "Expected privatizable type!");
7524
7525 // Collect all tail calls in the function as we cannot allow new allocas to
7526 // escape into tail recursion.
7527 // TODO: Be smarter about new allocas escaping into tail calls.
7529 bool UsedAssumedInformation = false;
7530 if (!A.checkForAllInstructions(
7531 [&](Instruction &I) {
7532 CallInst &CI = cast<CallInst>(I);
7533 if (CI.isTailCall())
7534 TailCalls.push_back(&CI);
7535 return true;
7536 },
7537 *this, {Instruction::Call}, UsedAssumedInformation))
7538 return ChangeStatus::UNCHANGED;
7539
7540 Argument *Arg = getAssociatedArgument();
7541 // Query AAAlign attribute for alignment of associated argument to
7542 // determine the best alignment of loads.
7543 const auto *AlignAA =
7544 A.getAAFor<AAAlign>(*this, IRPosition::value(*Arg), DepClassTy::NONE);
7545
7546 // Callback to repair the associated function. A new alloca is placed at the
7547 // beginning and initialized with the values passed through arguments. The
7548 // new alloca replaces the use of the old pointer argument.
7551 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7552 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7554 const DataLayout &DL = IP->getModule()->getDataLayout();
7555 unsigned AS = DL.getAllocaAddrSpace();
7556 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7557 Arg->getName() + ".priv", IP);
7558 createInitialization(*PrivatizableType, *AI, ReplacementFn,
7559 ArgIt->getArgNo(), IP);
7560
7561 if (AI->getType() != Arg->getType())
7562 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7563 AI, Arg->getType(), "", IP);
7564 Arg->replaceAllUsesWith(AI);
7565
7566 for (CallInst *CI : TailCalls)
7567 CI->setTailCall(false);
7568 };
7569
7570 // Callback to repair a call site of the associated function. The elements
7571 // of the privatizable type are loaded prior to the call and passed to the
7572 // new function version.
7575 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7576 // When no alignment is specified for the load instruction,
7577 // natural alignment is assumed.
7578 createReplacementValues(
7579 AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7580 *PrivatizableType, ACS,
7581 ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()),
7582 NewArgOperands);
7583 };
7584
7585 // Collect the types that will replace the privatizable type in the function
7586 // signature.
7587 SmallVector<Type *, 16> ReplacementTypes;
7588 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7589
7590 // Register a rewrite of the argument.
7591 if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,
7592 std::move(FnRepairCB),
7593 std::move(ACSRepairCB)))
7594 return ChangeStatus::CHANGED;
7595 return ChangeStatus::UNCHANGED;
7596 }
7597
7598 /// See AbstractAttribute::trackStatistics()
7599 void trackStatistics() const override {
7600 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7601 }
7602};
7603
7604struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7605 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7606 : AAPrivatizablePtrImpl(IRP, A) {}
7607
7608 /// See AbstractAttribute::initialize(...).
7609 void initialize(Attributor &A) override {
7610 // TODO: We can privatize more than arguments.
7611 indicatePessimisticFixpoint();
7612 }
7613
7614 ChangeStatus updateImpl(Attributor &A) override {
7615 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7616 "updateImpl will not be called");
7617 }
7618
7619 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7620 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7621 Value *Obj = getUnderlyingObject(&getAssociatedValue());
7622 if (!Obj) {
7623 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7624 return nullptr;
7625 }
7626
7627 if (auto *AI = dyn_cast<AllocaInst>(Obj))
7628 if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
7629 if (CI->isOne())
7630 return AI->getAllocatedType();
7631 if (auto *Arg = dyn_cast<Argument>(Obj)) {
7632 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7633 *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED);
7634 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7635 return PrivArgAA->getPrivatizableType();
7636 }
7637
7638 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7639 "alloca nor privatizable argument: "
7640 << *Obj << "!\n");
7641 return nullptr;
7642 }
7643
7644 /// See AbstractAttribute::trackStatistics()
7645 void trackStatistics() const override {
7646 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7647 }
7648};
7649
7650struct AAPrivatizablePtrCallSiteArgument final
7651 : public AAPrivatizablePtrFloating {
7652 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7653 : AAPrivatizablePtrFloating(IRP, A) {}
7654
7655 /// See AbstractAttribute::initialize(...).
7656 void initialize(Attributor &A) override {
7657 if (A.hasAttr(getIRPosition(), Attribute::ByVal))
7658 indicateOptimisticFixpoint();
7659 }
7660
7661 /// See AbstractAttribute::updateImpl(...).
7662 ChangeStatus updateImpl(Attributor &A) override {
7663 PrivatizableType = identifyPrivatizableType(A);
7664 if (!PrivatizableType)
7665 return ChangeStatus::UNCHANGED;
7666 if (!*PrivatizableType)
7667 return indicatePessimisticFixpoint();
7668
7669 const IRPosition &IRP = getIRPosition();
7670 bool IsKnownNoCapture;
7671 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
7672 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
7673 if (!IsAssumedNoCapture) {
7674 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7675 return indicatePessimisticFixpoint();
7676 }
7677
7678 bool IsKnownNoAlias;
7679 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7680 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
7681 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7682 return indicatePessimisticFixpoint();
7683 }
7684
7685 bool IsKnown;
7686 if (!AA::isAssumedReadOnly(A, IRP, *this, IsKnown)) {
7687 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7688 return indicatePessimisticFixpoint();
7689 }
7690
7691 return ChangeStatus::UNCHANGED;
7692 }
7693
7694 /// See AbstractAttribute::trackStatistics()
7695 void trackStatistics() const override {
7696 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7697 }
7698};
7699
7700struct AAPrivatizablePtrCallSiteReturned final
7701 : public AAPrivatizablePtrFloating {
7702 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7703 : AAPrivatizablePtrFloating(IRP, A) {}
7704
7705 /// See AbstractAttribute::initialize(...).
7706 void initialize(Attributor &A) override {
7707 // TODO: We can privatize more than arguments.
7708 indicatePessimisticFixpoint();
7709 }
7710
7711 /// See AbstractAttribute::trackStatistics()
7712 void trackStatistics() const override {
7713 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7714 }
7715};
7716
7717struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7718 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7719 : AAPrivatizablePtrFloating(IRP, A) {}
7720
7721 /// See AbstractAttribute::initialize(...).
7722 void initialize(Attributor &A) override {
7723 // TODO: We can privatize more than arguments.
7724 indicatePessimisticFixpoint();
7725 }
7726
7727 /// See AbstractAttribute::trackStatistics()
7728 void trackStatistics() const override {
7729 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7730 }
7731};
7732} // namespace
7733
7734/// -------------------- Memory Behavior Attributes ----------------------------
7735/// Includes read-none, read-only, and write-only.
7736/// ----------------------------------------------------------------------------
7737namespace {
7738struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7739 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7740 : AAMemoryBehavior(IRP, A) {}
7741
7742 /// See AbstractAttribute::initialize(...).
7743 void initialize(Attributor &A) override {
7744 intersectAssumedBits(BEST_STATE);
7745 getKnownStateFromValue(A, getIRPosition(), getState());
7746 AAMemoryBehavior::initialize(A);
7747 }
7748
7749 /// Return the memory behavior information encoded in the IR for \p IRP.
7750 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7751 BitIntegerState &State,
7752 bool IgnoreSubsumingPositions = false) {
7754 A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);
7755 for (const Attribute &Attr : Attrs) {
7756 switch (Attr.getKindAsEnum()) {
7757 case Attribute::ReadNone:
7758 State.addKnownBits(NO_ACCESSES);
7759 break;
7760 case Attribute::ReadOnly:
7761 State.addKnownBits(NO_WRITES);
7762 break;
7763 case Attribute::WriteOnly:
7764 State.addKnownBits(NO_READS);
7765 break;
7766 default:
7767 llvm_unreachable("Unexpected attribute!");
7768 }
7769 }
7770
7771 if (auto *I = dyn_cast<Instruction>(&IRP.getAnchorValue())) {
7772 if (!I->mayReadFromMemory())
7773 State.addKnownBits(NO_READS);
7774 if (!I->mayWriteToMemory())
7775 State.addKnownBits(NO_WRITES);
7776 }
7777 }
7778
7779 /// See AbstractAttribute::getDeducedAttributes(...).
7780 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7781 SmallVectorImpl<Attribute> &Attrs) const override {
7782 assert(Attrs.size() == 0);
7783 if (isAssumedReadNone())
7784 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
7785 else if (isAssumedReadOnly())
7786 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
7787 else if (isAssumedWriteOnly())
7788 Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
7789 assert(Attrs.size() <= 1);
7790 }
7791
7792 /// See AbstractAttribute::manifest(...).
7793 ChangeStatus manifest(Attributor &A) override {
7794 const IRPosition &IRP = getIRPosition();
7795
7796 if (A.hasAttr(IRP, Attribute::ReadNone,
7797 /* IgnoreSubsumingPositions */ true))
7798 return ChangeStatus::UNCHANGED;
7799
7800 // Check if we would improve the existing attributes first.
7801 SmallVector<Attribute, 4> DeducedAttrs;
7802 getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
7803 if (llvm::all_of(DeducedAttrs, [&](const Attribute &Attr) {
7804 return A.hasAttr(IRP, Attr.getKindAsEnum(),
7805 /* IgnoreSubsumingPositions */ true);
7806 }))
7807 return ChangeStatus::UNCHANGED;
7808
7809 // Clear existing attributes.
7810 A.removeAttrs(IRP, AttrKinds);
7811 // Clear conflicting writable attribute.
7812 if (isAssumedReadOnly())
7813 A.removeAttrs(IRP, Attribute::Writable);
7814
7815 // Use the generic manifest method.
7816 return IRAttribute::manifest(A);
7817 }
7818
7819 /// See AbstractState::getAsStr().
7820 const std::string getAsStr(Attributor *A) const override {
7821 if (isAssumedReadNone())
7822 return "readnone";
7823 if (isAssumedReadOnly())
7824 return "readonly";
7825 if (isAssumedWriteOnly())
7826 return "writeonly";
7827 return "may-read/write";
7828 }
7829
7830 /// The set of IR attributes AAMemoryBehavior deals with.
7831 static const Attribute::AttrKind AttrKinds[3];
7832};
7833
7834const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7835 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7836
7837/// Memory behavior attribute for a floating value.
7838struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7839 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7840 : AAMemoryBehaviorImpl(IRP, A) {}
7841
7842 /// See AbstractAttribute::updateImpl(...).
7843 ChangeStatus updateImpl(Attributor &A) override;
7844
7845 /// See AbstractAttribute::trackStatistics()
7846 void trackStatistics() const override {
7847 if (isAssumedReadNone())
7849 else if (isAssumedReadOnly())
7851 else if (isAssumedWriteOnly())
7853 }
7854
7855private:
7856 /// Return true if users of \p UserI might access the underlying
7857 /// variable/location described by \p U and should therefore be analyzed.
7858 bool followUsersOfUseIn(Attributor &A, const Use &U,
7859 const Instruction *UserI);
7860
7861 /// Update the state according to the effect of use \p U in \p UserI.
7862 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
7863};
7864
7865/// Memory behavior attribute for function argument.
7866struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
7867 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
7868 : AAMemoryBehaviorFloating(IRP, A) {}
7869
7870 /// See AbstractAttribute::initialize(...).
7871 void initialize(Attributor &A) override {
7872 intersectAssumedBits(BEST_STATE);
7873 const IRPosition &IRP = getIRPosition();
7874 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
7875 // can query it when we use has/getAttr. That would allow us to reuse the
7876 // initialize of the base class here.
7877 bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},
7878 /* IgnoreSubsumingPositions */ true);
7879 getKnownStateFromValue(A, IRP, getState(),
7880 /* IgnoreSubsumingPositions */ HasByVal);
7881 }
7882
7883 ChangeStatus manifest(Attributor &A) override {
7884 // TODO: Pointer arguments are not supported on vectors of pointers yet.
7885 if (!getAssociatedValue().getType()->isPointerTy())
7886 return ChangeStatus::UNCHANGED;
7887
7888 // TODO: From readattrs.ll: "inalloca parameters are always
7889 // considered written"
7890 if (A.hasAttr(getIRPosition(),
7891 {Attribute::InAlloca, Attribute::Preallocated})) {
7892 removeKnownBits(NO_WRITES);
7893 removeAssumedBits(NO_WRITES);
7894 }
7895 A.removeAttrs(getIRPosition(), AttrKinds);
7896 return AAMemoryBehaviorFloating::manifest(A);
7897 }
7898
7899 /// See AbstractAttribute::trackStatistics()
7900 void trackStatistics() const override {
7901 if (isAssumedReadNone())
7902 STATS_DECLTRACK_ARG_ATTR(readnone)
7903 else if (isAssumedReadOnly())
7904 STATS_DECLTRACK_ARG_ATTR(readonly)
7905 else if (isAssumedWriteOnly())
7906 STATS_DECLTRACK_ARG_ATTR(writeonly)
7907 }
7908};
7909
7910struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
7911 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
7912 : AAMemoryBehaviorArgument(IRP, A) {}
7913
7914 /// See AbstractAttribute::initialize(...).
7915 void initialize(Attributor &A) override {
7916 // If we don't have an associated attribute this is either a variadic call
7917 // or an indirect call, either way, nothing to do here.
7918 Argument *Arg = getAssociatedArgument();
7919 if (!Arg) {
7920 indicatePessimisticFixpoint();
7921 return;
7922 }
7923 if (Arg->hasByValAttr()) {
7924 addKnownBits(NO_WRITES);
7925 removeKnownBits(NO_READS);
7926 removeAssumedBits(NO_READS);
7927 }
7928 AAMemoryBehaviorArgument::initialize(A);
7929 if (getAssociatedFunction()->isDeclaration())
7930 indicatePessimisticFixpoint();
7931 }
7932
7933 /// See AbstractAttribute::updateImpl(...).
7934 ChangeStatus updateImpl(Attributor &A) override {
7935 // TODO: Once we have call site specific value information we can provide
7936 // call site specific liveness liveness information and then it makes
7937 // sense to specialize attributes for call sites arguments instead of
7938 // redirecting requests to the callee argument.
7939 Argument *Arg = getAssociatedArgument();
7940 const IRPosition &ArgPos = IRPosition::argument(*Arg);
7941 auto *ArgAA =
7942 A.getAAFor<AAMemoryBehavior>(*this, ArgPos, DepClassTy::REQUIRED);
7943 if (!ArgAA)
7944 return indicatePessimisticFixpoint();
7945 return clampStateAndIndicateChange(getState(), ArgAA->getState());
7946 }
7947
7948 /// See AbstractAttribute::trackStatistics()
7949 void trackStatistics() const override {
7950 if (isAssumedReadNone())
7952 else if (isAssumedReadOnly())
7954 else if (isAssumedWriteOnly())
7956 }
7957};
7958
7959/// Memory behavior attribute for a call site return position.
7960struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
7961 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
7962 : AAMemoryBehaviorFloating(IRP, A) {}
7963
7964 /// See AbstractAttribute::initialize(...).
7965 void initialize(Attributor &A) override {
7966 AAMemoryBehaviorImpl::initialize(A);
7967 }
7968 /// See AbstractAttribute::manifest(...).
7969 ChangeStatus manifest(Attributor &A) override {
7970 // We do not annotate returned values.
7971 return ChangeStatus::UNCHANGED;
7972 }
7973
7974 /// See AbstractAttribute::trackStatistics()
7975 void trackStatistics() const override {}
7976};
7977
7978/// An AA to represent the memory behavior function attributes.
7979struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
7980 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
7981 : AAMemoryBehaviorImpl(IRP, A) {}
7982
7983 /// See AbstractAttribute::updateImpl(Attributor &A).
7984 ChangeStatus updateImpl(Attributor &A) override;
7985
7986 /// See AbstractAttribute::manifest(...).
7987 ChangeStatus manifest(Attributor &A) override {
7988 // TODO: It would be better to merge this with AAMemoryLocation, so that
7989 // we could determine read/write per location. This would also have the
7990 // benefit of only one place trying to manifest the memory attribute.
7991 Function &F = cast<Function>(getAnchorValue());
7993 if (isAssumedReadNone())
7994 ME = MemoryEffects::none();
7995 else if (isAssumedReadOnly())
7997 else if (isAssumedWriteOnly())
7999
8000 A.removeAttrs(getIRPosition(), AttrKinds);
8001 // Clear conflicting writable attribute.
8002 if (ME.onlyReadsMemory())
8003 for (Argument &Arg : F.args())
8004 A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
8005 return A.manifestAttrs(getIRPosition(),
8006 Attribute::getWithMemoryEffects(F.getContext(), ME));
8007 }
8008
8009 /// See AbstractAttribute::trackStatistics()
8010 void trackStatistics() const override {
8011 if (isAssumedReadNone())
8012 STATS_DECLTRACK_FN_ATTR(readnone)
8013 else if (isAssumedReadOnly())
8014 STATS_DECLTRACK_FN_ATTR(readonly)
8015 else if (isAssumedWriteOnly())
8016 STATS_DECLTRACK_FN_ATTR(writeonly)
8017 }
8018};
8019
8020/// AAMemoryBehavior attribute for call sites.
8021struct AAMemoryBehaviorCallSite final
8022 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8023 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8024 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8025
8026 /// See AbstractAttribute::manifest(...).
8027 ChangeStatus manifest(Attributor &A) override {
8028 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8029 CallBase &CB = cast<CallBase>(getAnchorValue());
8031 if (isAssumedReadNone())
8032 ME = MemoryEffects::none();
8033 else if (isAssumedReadOnly())
8035 else if (isAssumedWriteOnly())
8037
8038 A.removeAttrs(getIRPosition(), AttrKinds);
8039 // Clear conflicting writable attribute.
8040 if (ME.onlyReadsMemory())
8041 for (Use &U : CB.args())
8042 A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),
8043 Attribute::Writable);
8044 return A.manifestAttrs(
8045 getIRPosition(), Attribute::getWithMemoryEffects(CB.getContext(), ME));
8046 }
8047
8048 /// See AbstractAttribute::trackStatistics()
8049 void trackStatistics() const override {
8050 if (isAssumedReadNone())
8051 STATS_DECLTRACK_CS_ATTR(readnone)
8052 else if (isAssumedReadOnly())
8053 STATS_DECLTRACK_CS_ATTR(readonly)
8054 else if (isAssumedWriteOnly())
8055 STATS_DECLTRACK_CS_ATTR(writeonly)
8056 }
8057};
8058
8059ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8060
8061 // The current assumed state used to determine a change.
8062 auto AssumedState = getAssumed();
8063
8064 auto CheckRWInst = [&](Instruction &I) {
8065 // If the instruction has an own memory behavior state, use it to restrict
8066 // the local state. No further analysis is required as the other memory
8067 // state is as optimistic as it gets.
8068 if (const auto *CB = dyn_cast<CallBase>(&I)) {
8069 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8071 if (MemBehaviorAA) {
8072 intersectAssumedBits(MemBehaviorAA->getAssumed());
8073 return !isAtFixpoint();
8074 }
8075 }
8076
8077 // Remove access kind modifiers if necessary.
8078 if (I.mayReadFromMemory())
8079 removeAssumedBits(NO_READS);
8080 if (I.mayWriteToMemory())
8081 removeAssumedBits(NO_WRITES);
8082 return !isAtFixpoint();
8083 };
8084
8085 bool UsedAssumedInformation = false;
8086 if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
8087 UsedAssumedInformation))
8088 return indicatePessimisticFixpoint();
8089
8090 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8092}
8093
8094ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8095
8096 const IRPosition &IRP = getIRPosition();
8097 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8098 AAMemoryBehavior::StateType &S = getState();
8099
8100 // First, check the function scope. We take the known information and we avoid
8101 // work if the assumed information implies the current assumed information for
8102 // this attribute. This is a valid for all but byval arguments.
8103 Argument *Arg = IRP.getAssociatedArgument();
8104 AAMemoryBehavior::base_t FnMemAssumedState =
8106 if (!Arg || !Arg->hasByValAttr()) {
8107 const auto *FnMemAA =
8108 A.getAAFor<AAMemoryBehavior>(*this, FnPos, DepClassTy::OPTIONAL);
8109 if (FnMemAA) {
8110 FnMemAssumedState = FnMemAA->getAssumed();
8111 S.addKnownBits(FnMemAA->getKnown());
8112 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8114 }
8115 }
8116
8117 // The current assumed state used to determine a change.
8118 auto AssumedState = S.getAssumed();
8119
8120 // Make sure the value is not captured (except through "return"), if
8121 // it is, any information derived would be irrelevant anyway as we cannot
8122 // check the potential aliases introduced by the capture. However, no need
8123 // to fall back to anythign less optimistic than the function state.
8124 bool IsKnownNoCapture;
8125 const AANoCapture *ArgNoCaptureAA = nullptr;
8126 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
8127 A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
8128 &ArgNoCaptureAA);
8129
8130 if (!IsAssumedNoCapture &&
8131 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8132 S.intersectAssumedBits(FnMemAssumedState);
8133 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8135 }
8136
8137 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8138 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8139 Instruction *UserI = cast<Instruction>(U.getUser());
8140 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8141 << " \n");
8142
8143 // Droppable users, e.g., llvm::assume does not actually perform any action.
8144 if (UserI->isDroppable())
8145 return true;
8146
8147 // Check if the users of UserI should also be visited.
8148 Follow = followUsersOfUseIn(A, U, UserI);
8149
8150 // If UserI might touch memory we analyze the use in detail.
8151 if (UserI->mayReadOrWriteMemory())
8152 analyzeUseIn(A, U, UserI);
8153
8154 return !isAtFixpoint();
8155 };
8156
8157 if (!A.checkForAllUses(UsePred, *this, getAssociatedValue()))
8158 return indicatePessimisticFixpoint();
8159
8160 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8162}
8163
8164bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8165 const Instruction *UserI) {
8166 // The loaded value is unrelated to the pointer argument, no need to
8167 // follow the users of the load.
8168 if (isa<LoadInst>(UserI) || isa<ReturnInst>(UserI))
8169 return false;
8170
8171 // By default we follow all uses assuming UserI might leak information on U,
8172 // we have special handling for call sites operands though.
8173 const auto *CB = dyn_cast<CallBase>(UserI);
8174 if (!CB || !CB->isArgOperand(&U))
8175 return true;
8176
8177 // If the use is a call argument known not to be captured, the users of
8178 // the call do not need to be visited because they have to be unrelated to
8179 // the input. Note that this check is not trivial even though we disallow
8180 // general capturing of the underlying argument. The reason is that the
8181 // call might the argument "through return", which we allow and for which we
8182 // need to check call users.
8183 if (U.get()->getType()->isPointerTy()) {
8184 unsigned ArgNo = CB->getArgOperandNo(&U);
8185 bool IsKnownNoCapture;
8186 return !AA::hasAssumedIRAttr<Attribute::NoCapture>(
8187 A, this, IRPosition::callsite_argument(*CB, ArgNo),
8188 DepClassTy::OPTIONAL, IsKnownNoCapture);
8189 }
8190
8191 return true;
8192}
8193
8194void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8195 const Instruction *UserI) {
8196 assert(UserI->mayReadOrWriteMemory());
8197
8198 switch (UserI->getOpcode()) {
8199 default:
8200 // TODO: Handle all atomics and other side-effect operations we know of.
8201 break;
8202 case Instruction::Load:
8203 // Loads cause the NO_READS property to disappear.
8204 removeAssumedBits(NO_READS);
8205 return;
8206
8207 case Instruction::Store:
8208 // Stores cause the NO_WRITES property to disappear if the use is the
8209 // pointer operand. Note that while capturing was taken care of somewhere
8210 // else we need to deal with stores of the value that is not looked through.
8211 if (cast<StoreInst>(UserI)->getPointerOperand() == U.get())
8212 removeAssumedBits(NO_WRITES);
8213 else
8214 indicatePessimisticFixpoint();
8215 return;
8216
8217 case Instruction::Call:
8218 case Instruction::CallBr:
8219 case Instruction::Invoke: {
8220 // For call sites we look at the argument memory behavior attribute (this
8221 // could be recursive!) in order to restrict our own state.
8222 const auto *CB = cast<CallBase>(UserI);
8223
8224 // Give up on operand bundles.
8225 if (CB->isBundleOperand(&U)) {
8226 indicatePessimisticFixpoint();
8227 return;
8228 }
8229
8230 // Calling a function does read the function pointer, maybe write it if the
8231 // function is self-modifying.
8232 if (CB->isCallee(&U)) {
8233 removeAssumedBits(NO_READS);
8234 break;
8235 }
8236
8237 // Adjust the possible access behavior based on the information on the
8238 // argument.
8239 IRPosition Pos;
8240 if (U.get()->getType()->isPointerTy())
8242 else
8244 const auto *MemBehaviorAA =
8245 A.getAAFor<AAMemoryBehavior>(*this, Pos, DepClassTy::OPTIONAL);
8246 if (!MemBehaviorAA)
8247 break;
8248 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8249 // and at least "known".
8250 intersectAssumedBits(MemBehaviorAA->getAssumed());
8251 return;
8252 }
8253 };
8254
8255 // Generally, look at the "may-properties" and adjust the assumed state if we
8256 // did not trigger special handling before.
8257 if (UserI->mayReadFromMemory())
8258 removeAssumedBits(NO_READS);
8259 if (UserI->mayWriteToMemory())
8260 removeAssumedBits(NO_WRITES);
8261}
8262} // namespace
8263
8264/// -------------------- Memory Locations Attributes ---------------------------
8265/// Includes read-none, argmemonly, inaccessiblememonly,
8266/// inaccessiblememorargmemonly
8267/// ----------------------------------------------------------------------------
8268
8271 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8272 return "all memory";
8274 return "no memory";
8275 std::string S = "memory:";
8276 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8277 S += "stack,";
8278 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8279 S += "constant,";
8281 S += "internal global,";
8283 S += "external global,";
8284 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8285 S += "argument,";
8287 S += "inaccessible,";
8288 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8289 S += "malloced,";
8290 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8291 S += "unknown,";
8292 S.pop_back();
8293 return S;
8294}
8295
8296namespace {
8297struct AAMemoryLocationImpl : public AAMemoryLocation {
8298
8299 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8301 AccessKind2Accesses.fill(nullptr);
8302 }
8303
8304 ~AAMemoryLocationImpl() {
8305 // The AccessSets are allocated via a BumpPtrAllocator, we call
8306 // the destructor manually.
8307 for (AccessSet *AS : AccessKind2Accesses)
8308 if (AS)
8309 AS->~AccessSet();
8310 }
8311
8312 /// See AbstractAttribute::initialize(...).
8313 void initialize(Attributor &A) override {
8314 intersectAssumedBits(BEST_STATE);
8315 getKnownStateFromValue(A, getIRPosition(), getState());
8316 AAMemoryLocation::initialize(A);
8317 }
8318
8319 /// Return the memory behavior information encoded in the IR for \p IRP.
8320 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8321 BitIntegerState &State,
8322 bool IgnoreSubsumingPositions = false) {
8323 // For internal functions we ignore `argmemonly` and
8324 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8325 // constant propagation. It is unclear if this is the best way but it is
8326 // unlikely this will cause real performance problems. If we are deriving
8327 // attributes for the anchor function we even remove the attribute in
8328 // addition to ignoring it.
8329 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8330 // MemoryEffects::Other as a possible location.
8331 bool UseArgMemOnly = true;
8332 Function *AnchorFn = IRP.getAnchorScope();
8333 if (AnchorFn && A.isRunOn(*AnchorFn))
8334 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8335
8337 A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8338 for (const Attribute &Attr : Attrs) {
8339 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8340 MemoryEffects ME = Attr.getMemoryEffects();
8341 if (ME.doesNotAccessMemory()) {
8342 State.addKnownBits(NO_LOCAL_MEM | NO_CONST_MEM);
8343 continue;
8344 }
8345 if (ME.onlyAccessesInaccessibleMem()) {
8346 State.addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
8347 continue;
8348 }
8349 if (ME.onlyAccessesArgPointees()) {
8350 if (UseArgMemOnly)
8351 State.addKnownBits(inverseLocation(NO_ARGUMENT_MEM, true, true));
8352 else {
8353 // Remove location information, only keep read/write info.
8354 ME = MemoryEffects(ME.getModRef());
8355 A.manifestAttrs(IRP,
8357 IRP.getAnchorValue().getContext(), ME),
8358 /*ForceReplace*/ true);
8359 }
8360 continue;
8361 }
8363 if (UseArgMemOnly)
8364 State.addKnownBits(inverseLocation(
8365 NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
8366 else {
8367 // Remove location information, only keep read/write info.
8368 ME = MemoryEffects(ME.getModRef());
8369 A.manifestAttrs(IRP,
8371 IRP.getAnchorValue().getContext(), ME),
8372 /*ForceReplace*/ true);
8373 }
8374 continue;
8375 }
8376 }
8377 }
8378
8379 /// See AbstractAttribute::getDeducedAttributes(...).
8380 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8381 SmallVectorImpl<Attribute> &Attrs) const override {
8382 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8383 assert(Attrs.size() == 0);
8384 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8385 if (isAssumedReadNone())
8386 Attrs.push_back(
8388 else if (isAssumedInaccessibleMemOnly())
8391 else if (isAssumedArgMemOnly())
8392 Attrs.push_back(
8394 else if (isAssumedInaccessibleOrArgMemOnly())
8397 }
8398 assert(Attrs.size() <= 1);
8399 }
8400
8401 /// See AbstractAttribute::manifest(...).
8402 ChangeStatus manifest(Attributor &A) override {
8403 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8404 // provide per-location modref information here.
8405 const IRPosition &IRP = getIRPosition();
8406
8407 SmallVector<Attribute, 1> DeducedAttrs;
8408 getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
8409 if (DeducedAttrs.size() != 1)
8410 return ChangeStatus::UNCHANGED;
8411 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8412
8413 return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(
8414 IRP.getAnchorValue().getContext(), ME));
8415 }
8416
8417 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8418 bool checkForAllAccessesToMemoryKind(
8419 function_ref<bool(const Instruction *, const Value *, AccessKind,
8420 MemoryLocationsKind)>
8421 Pred,
8422 MemoryLocationsKind RequestedMLK) const override {
8423 if (!isValidState())
8424 return false;
8425
8426 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8427 if (AssumedMLK == NO_LOCATIONS)
8428 return true;
8429
8430 unsigned Idx = 0;
8431 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8432 CurMLK *= 2, ++Idx) {
8433 if (CurMLK & RequestedMLK)
8434 continue;
8435
8436 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8437 for (const AccessInfo &AI : *Accesses)
8438 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8439 return false;
8440 }
8441
8442 return true;
8443 }
8444
8445 ChangeStatus indicatePessimisticFixpoint() override {
8446 // If we give up and indicate a pessimistic fixpoint this instruction will
8447 // become an access for all potential access kinds:
8448 // TODO: Add pointers for argmemonly and globals to improve the results of
8449 // checkForAllAccessesToMemoryKind.
8450 bool Changed = false;
8451 MemoryLocationsKind KnownMLK = getKnown();
8452 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
8453 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8454 if (!(CurMLK & KnownMLK))
8455 updateStateAndAccessesMap(getState(), CurMLK, I, nullptr, Changed,
8456 getAccessKindFromInst(I));
8457 return AAMemoryLocation::indicatePessimisticFixpoint();
8458 }
8459
8460protected:
8461 /// Helper struct to tie together an instruction that has a read or write
8462 /// effect with the pointer it accesses (if any).
8463 struct AccessInfo {
8464
8465 /// The instruction that caused the access.
8466 const Instruction *I;
8467
8468 /// The base pointer that is accessed, or null if unknown.
8469 const Value *Ptr;
8470
8471 /// The kind of access (read/write/read+write).
8473
8474 bool operator==(const AccessInfo &RHS) const {
8475 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8476 }
8477 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8478 if (LHS.I != RHS.I)
8479 return LHS.I < RHS.I;
8480 if (LHS.Ptr != RHS.Ptr)
8481 return LHS.Ptr < RHS.Ptr;
8482 if (LHS.Kind != RHS.Kind)
8483 return LHS.Kind < RHS.Kind;
8484 return false;
8485 }
8486 };
8487
8488 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8489 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8490 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8491 std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses;
8492
8493 /// Categorize the pointer arguments of CB that might access memory in
8494 /// AccessedLoc and update the state and access map accordingly.
8495 void
8496 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8497 AAMemoryLocation::StateType &AccessedLocs,
8498 bool &Changed);
8499
8500 /// Return the kind(s) of location that may be accessed by \p V.
8502 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8503
8504 /// Return the access kind as determined by \p I.
8505 AccessKind getAccessKindFromInst(const Instruction *I) {
8506 AccessKind AK = READ_WRITE;
8507 if (I) {
8508 AK = I->mayReadFromMemory() ? READ : NONE;
8509 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8510 }
8511 return AK;
8512 }
8513
8514 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8515 /// an access of kind \p AK to a \p MLK memory location with the access
8516 /// pointer \p Ptr.
8517 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8518 MemoryLocationsKind MLK, const Instruction *I,
8519 const Value *Ptr, bool &Changed,
8520 AccessKind AK = READ_WRITE) {
8521
8522 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8523 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(MLK)];
8524 if (!Accesses)
8525 Accesses = new (Allocator) AccessSet();
8526 Changed |= Accesses->insert(AccessInfo{I, Ptr, AK}).second;
8527 if (MLK == NO_UNKOWN_MEM)
8528 MLK = NO_LOCATIONS;
8529 State.removeAssumedBits(MLK);
8530 }
8531
8532 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8533 /// arguments, and update the state and access map accordingly.
8534 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8535 AAMemoryLocation::StateType &State, bool &Changed,
8536 unsigned AccessAS = 0);
8537
8538 /// Used to allocate access sets.
8540};
8541
8542void AAMemoryLocationImpl::categorizePtrValue(
8543 Attributor &A, const Instruction &I, const Value &Ptr,
8544 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8545 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8546 << Ptr << " ["
8547 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8548
8549 auto Pred = [&](Value &Obj) {
8550 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8551 // TODO: recognize the TBAA used for constant accesses.
8552 MemoryLocationsKind MLK = NO_LOCATIONS;
8553
8554 // Filter accesses to constant (GPU) memory if we have an AS at the access
8555 // site or the object is known to actually have the associated AS.
8556 if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
8557 (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
8558 isIdentifiedObject(&Obj))) &&
8559 AA::isGPU(*I.getModule()))
8560 return true;
8561
8562 if (isa<UndefValue>(&Obj))
8563 return true;
8564 if (isa<Argument>(&Obj)) {
8565 // TODO: For now we do not treat byval arguments as local copies performed
8566 // on the call edge, though, we should. To make that happen we need to
8567 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8568 // would also allow us to mark functions only accessing byval arguments as
8569 // readnone again, arguably their accesses have no effect outside of the
8570 // function, like accesses to allocas.
8571 MLK = NO_ARGUMENT_MEM;
8572 } else if (auto *GV = dyn_cast<GlobalValue>(&Obj)) {
8573 // Reading constant memory is not treated as a read "effect" by the
8574 // function attr pass so we won't neither. Constants defined by TBAA are
8575 // similar. (We know we do not write it because it is constant.)
8576 if (auto *GVar = dyn_cast<GlobalVariable>(GV))
8577 if (GVar->isConstant())
8578 return true;
8579
8580 if (GV->hasLocalLinkage())
8581 MLK = NO_GLOBAL_INTERNAL_MEM;
8582 else
8583 MLK = NO_GLOBAL_EXTERNAL_MEM;
8584 } else if (isa<ConstantPointerNull>(&Obj) &&
8585 (!NullPointerIsDefined(getAssociatedFunction(), AccessAS) ||
8586 !NullPointerIsDefined(getAssociatedFunction(), ObjectAS))) {
8587 return true;
8588 } else if (isa<AllocaInst>(&Obj)) {
8589 MLK = NO_LOCAL_MEM;
8590 } else if (const auto *CB = dyn_cast<CallBase>(&Obj)) {
8591 bool IsKnownNoAlias;
8592 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8594 IsKnownNoAlias))
8595 MLK = NO_MALLOCED_MEM;
8596 else
8597 MLK = NO_UNKOWN_MEM;
8598 } else {
8599 MLK = NO_UNKOWN_MEM;
8600 }
8601
8602 assert(MLK != NO_LOCATIONS && "No location specified!");
8603 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8604 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8605 updateStateAndAccessesMap(State, MLK, &I, &Obj, Changed,
8606 getAccessKindFromInst(&I));
8607
8608 return true;
8609 };
8610
8611 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8613 if (!AA || !AA->forallUnderlyingObjects(Pred, AA::Intraprocedural)) {
8614 LLVM_DEBUG(
8615 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8616 updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed,
8617 getAccessKindFromInst(&I));
8618 return;
8619 }
8620
8621 LLVM_DEBUG(
8622 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8623 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8624}
8625
8626void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8627 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8628 bool &Changed) {
8629 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8630
8631 // Skip non-pointer arguments.
8632 const Value *ArgOp = CB.getArgOperand(ArgNo);
8633 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8634 continue;
8635
8636 // Skip readnone arguments.
8637 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8638 const auto *ArgOpMemLocationAA =
8639 A.getAAFor<AAMemoryBehavior>(*this, ArgOpIRP, DepClassTy::OPTIONAL);
8640
8641 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8642 continue;
8643
8644 // Categorize potentially accessed pointer arguments as if there was an
8645 // access instruction with them as pointer.
8646 categorizePtrValue(A, CB, *ArgOp, AccessedLocs, Changed);
8647 }
8648}
8649
8651AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8652 bool &Changed) {
8653 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8654 << I << "\n");
8655
8656 AAMemoryLocation::StateType AccessedLocs;
8657 AccessedLocs.intersectAssumedBits(NO_LOCATIONS);
8658
8659 if (auto *CB = dyn_cast<CallBase>(&I)) {
8660
8661 // First check if we assume any memory is access is visible.
8662 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8664 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8665 << " [" << CBMemLocationAA << "]\n");
8666 if (!CBMemLocationAA) {
8667 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr,
8668 Changed, getAccessKindFromInst(&I));
8669 return NO_UNKOWN_MEM;
8670 }
8671
8672 if (CBMemLocationAA->isAssumedReadNone())
8673 return NO_LOCATIONS;
8674
8675 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8676 updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &I, nullptr,
8677 Changed, getAccessKindFromInst(&I));
8678 return AccessedLocs.getAssumed();
8679 }
8680
8681 uint32_t CBAssumedNotAccessedLocs =
8682 CBMemLocationAA->getAssumedNotAccessedLocation();
8683
8684 // Set the argmemonly and global bit as we handle them separately below.
8685 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8686 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8687
8688 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8689 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8690 continue;
8691 updateStateAndAccessesMap(AccessedLocs, CurMLK, &I, nullptr, Changed,
8692 getAccessKindFromInst(&I));
8693 }
8694
8695 // Now handle global memory if it might be accessed. This is slightly tricky
8696 // as NO_GLOBAL_MEM has multiple bits set.
8697 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8698 if (HasGlobalAccesses) {
8699 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8700 AccessKind Kind, MemoryLocationsKind MLK) {
8701 updateStateAndAccessesMap(AccessedLocs, MLK, &I, Ptr, Changed,
8702 getAccessKindFromInst(&I));
8703 return true;
8704 };
8705 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8706 AccessPred, inverseLocation(NO_GLOBAL_MEM, false, false)))
8707 return AccessedLocs.getWorstState();
8708 }
8709
8710 LLVM_DEBUG(
8711 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8712 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8713
8714 // Now handle argument memory if it might be accessed.
8715 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8716 if (HasArgAccesses)
8717 categorizeArgumentPointerLocations(A, *CB, AccessedLocs, Changed);
8718
8719 LLVM_DEBUG(
8720 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8721 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8722
8723 return AccessedLocs.getAssumed();
8724 }
8725
8726 if (const Value *Ptr = getPointerOperand(&I, /* AllowVolatile */ true)) {
8727 LLVM_DEBUG(
8728 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8729 << I << " [" << *Ptr << "]\n");
8730 categorizePtrValue(A, I, *Ptr, AccessedLocs, Changed,
8731 Ptr->getType()->getPointerAddressSpace());
8732 return AccessedLocs.getAssumed();
8733 }
8734
8735 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8736 << I << "\n");
8737 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr, Changed,
8738 getAccessKindFromInst(&I));
8739 return AccessedLocs.getAssumed();
8740}
8741
8742/// An AA to represent the memory behavior function attributes.
8743struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8744 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8745 : AAMemoryLocationImpl(IRP, A) {}
8746
8747 /// See AbstractAttribute::updateImpl(Attributor &A).
8748 ChangeStatus updateImpl(Attributor &A) override {
8749
8750 const auto *MemBehaviorAA =
8751 A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
8752 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8753 if (MemBehaviorAA->isKnownReadNone())
8754 return indicateOptimisticFixpoint();
8756 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8757 A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
8758 return ChangeStatus::UNCHANGED;
8759 }
8760
8761 // The current assumed state used to determine a change.
8762 auto AssumedState = getAssumed();
8763 bool Changed = false;
8764
8765 auto CheckRWInst = [&](Instruction &I) {
8766 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8767 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8768 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8769 removeAssumedBits(inverseLocation(MLK, false, false));
8770 // Stop once only the valid bit set in the *not assumed location*, thus
8771 // once we don't actually exclude any memory locations in the state.
8772 return getAssumedNotAccessedLocation() != VALID_STATE;
8773 };
8774
8775 bool UsedAssumedInformation = false;
8776 if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
8777 UsedAssumedInformation))
8778 return indicatePessimisticFixpoint();
8779
8780 Changed |= AssumedState != getAssumed();
8781 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8782 }
8783
8784 /// See AbstractAttribute::trackStatistics()
8785 void trackStatistics() const override {
8786 if (isAssumedReadNone())
8787 STATS_DECLTRACK_FN_ATTR(readnone)
8788 else if (isAssumedArgMemOnly())
8789 STATS_DECLTRACK_FN_ATTR(argmemonly)
8790 else if (isAssumedInaccessibleMemOnly())
8791 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8792 else if (isAssumedInaccessibleOrArgMemOnly())
8793 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8794 }
8795};
8796
8797/// AAMemoryLocation attribute for call sites.
8798struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8799 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8800 : AAMemoryLocationImpl(IRP, A) {}
8801
8802 /// See AbstractAttribute::updateImpl(...).
8803 ChangeStatus updateImpl(Attributor &A) override {
8804 // TODO: Once we have call site specific value information we can provide
8805 // call site specific liveness liveness information and then it makes
8806 // sense to specialize attributes for call sites arguments instead of
8807 // redirecting requests to the callee argument.
8808 Function *F = getAssociatedFunction();
8809 const IRPosition &FnPos = IRPosition::function(*F);
8810 auto *FnAA =
8811 A.getAAFor<AAMemoryLocation>(*this, FnPos, DepClassTy::REQUIRED);
8812 if (!FnAA)
8813 return indicatePessimisticFixpoint();
8814 bool Changed = false;
8815 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8816 AccessKind Kind, MemoryLocationsKind MLK) {
8817 updateStateAndAccessesMap(getState(), MLK, I, Ptr, Changed,
8818 getAccessKindFromInst(I));
8819 return true;
8820 };
8821 if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))
8822 return indicatePessimisticFixpoint();
8823 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8824 }
8825
8826 /// See AbstractAttribute::trackStatistics()
8827 void trackStatistics() const override {
8828 if (isAssumedReadNone())
8829 STATS_DECLTRACK_CS_ATTR(readnone)
8830 }
8831};
8832} // namespace
8833
8834/// ------------------ denormal-fp-math Attribute -------------------------
8835
8836namespace {
8837struct AADenormalFPMathImpl : public AADenormalFPMath {
8838 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8839 : AADenormalFPMath(IRP, A) {}
8840
8841 const std::string getAsStr(Attributor *A) const override {
8842 std::string Str("AADenormalFPMath[");
8844
8845 DenormalState Known = getKnown();
8846 if (Known.Mode.isValid())
8847 OS << "denormal-fp-math=" << Known.Mode;
8848 else
8849 OS << "invalid";
8850
8851 if (Known.ModeF32.isValid())
8852 OS << " denormal-fp-math-f32=" << Known.ModeF32;
8853 OS << ']';
8854 return OS.str();
8855 }
8856};
8857
8858struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
8859 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
8860 : AADenormalFPMathImpl(IRP, A) {}
8861
8862 void initialize(Attributor &A) override {
8863 const Function *F = getAnchorScope();
8864 DenormalMode Mode = F->getDenormalModeRaw();
8865 DenormalMode ModeF32 = F->getDenormalModeF32Raw();
8866
8867 // TODO: Handling this here prevents handling the case where a callee has a
8868 // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from
8869 // a function with a fully fixed mode.
8870 if (ModeF32 == DenormalMode::getInvalid())
8871 ModeF32 = Mode;
8872 Known = DenormalState{Mode, ModeF32};
8873 if (isModeFixed())
8874 indicateFixpoint();
8875 }
8876
8877 ChangeStatus updateImpl(Attributor &A) override {
8878 ChangeStatus Change = ChangeStatus::UNCHANGED;
8879
8880 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
8881 Function *Caller = CS.getInstruction()->getFunction();
8882 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
8883 << "->" << getAssociatedFunction()->getName() << '\n');
8884
8885 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
8886 *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
8887 if (!CallerInfo)
8888 return false;
8889
8890 Change = Change | clampStateAndIndicateChange(this->getState(),
8891 CallerInfo->getState());
8892 return true;
8893 };
8894
8895 bool AllCallSitesKnown = true;
8896 if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))
8897 return indicatePessimisticFixpoint();
8898
8899 if (Change == ChangeStatus::CHANGED && isModeFixed())
8900 indicateFixpoint();
8901 return Change;
8902 }
8903
8904 ChangeStatus manifest(Attributor &A) override {
8905 LLVMContext &Ctx = getAssociatedFunction()->getContext();
8906
8907 SmallVector<Attribute, 2> AttrToAdd;
8908 SmallVector<StringRef, 2> AttrToRemove;
8909 if (Known.Mode == DenormalMode::getDefault()) {
8910 AttrToRemove.push_back("denormal-fp-math");
8911 } else {
8912 AttrToAdd.push_back(
8913 Attribute::get(Ctx, "denormal-fp-math", Known.Mode.str()));
8914 }
8915
8916 if (Known.ModeF32 != Known.Mode) {
8917 AttrToAdd.push_back(
8918 Attribute::get(Ctx, "denormal-fp-math-f32", Known.ModeF32.str()));
8919 } else {
8920 AttrToRemove.push_back("denormal-fp-math-f32");
8921 }
8922
8923 auto &IRP = getIRPosition();
8924
8925 // TODO: There should be a combined add and remove API.
8926 return A.removeAttrs(IRP, AttrToRemove) |
8927 A.manifestAttrs(IRP, AttrToAdd, /*ForceReplace=*/true);
8928 }
8929
8930 void trackStatistics() const override {
8931 STATS_DECLTRACK_FN_ATTR(denormal_fp_math)
8932 }
8933};
8934} // namespace
8935
8936/// ------------------ Value Constant Range Attribute -------------------------
8937
8938namespace {
8939struct AAValueConstantRangeImpl : AAValueConstantRange {
8940 using StateType = IntegerRangeState;
8941 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
8942 : AAValueConstantRange(IRP, A) {}
8943
8944 /// See AbstractAttribute::initialize(..).
8945 void initialize(Attributor &A) override {
8946 if (A.hasSimplificationCallback(getIRPosition())) {
8947 indicatePessimisticFixpoint();
8948 return;
8949 }
8950
8951 // Intersect a range given by SCEV.
8952 intersectKnown(getConstantRangeFromSCEV(A, getCtxI()));
8953
8954 // Intersect a range given by LVI.
8955 intersectKnown(getConstantRangeFromLVI(A, getCtxI()));
8956 }
8957
8958 /// See AbstractAttribute::getAsStr().
8959 const std::string getAsStr(Attributor *A) const override {
8960 std::string Str;
8962 OS << "range(" << getBitWidth() << ")<";
8963 getKnown().print(OS);
8964 OS << " / ";
8965 getAssumed().print(OS);
8966 OS << ">";
8967 return OS.str();
8968 }
8969
8970 /// Helper function to get a SCEV expr for the associated value at program
8971 /// point \p I.
8972 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
8973 if (!getAnchorScope())
8974 return nullptr;
8975
8976 ScalarEvolution *SE =
8977 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
8978 *getAnchorScope());
8979
8980 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
8981 *getAnchorScope());
8982
8983 if (!SE || !LI)
8984 return nullptr;
8985
8986 const SCEV *S = SE->getSCEV(&getAssociatedValue());
8987 if (!I)
8988 return S;
8989
8990 return SE->getSCEVAtScope(S, LI->getLoopFor(I->getParent()));
8991 }
8992
8993 /// Helper function to get a range from SCEV for the associated value at
8994 /// program point \p I.
8995 ConstantRange getConstantRangeFromSCEV(Attributor &A,
8996 const Instruction *I = nullptr) const {
8997 if (!getAnchorScope())
8998 return getWorstState(getBitWidth());
8999
9000 ScalarEvolution *SE =
9001 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9002 *getAnchorScope());
9003
9004 const SCEV *S = getSCEV(A, I);
9005 if (!SE || !S)
9006 return getWorstState(getBitWidth());
9007
9008 return SE->getUnsignedRange(S);
9009 }
9010
9011 /// Helper function to get a range from LVI for the associated value at
9012 /// program point \p I.
9014 getConstantRangeFromLVI(Attributor &A,
9015 const Instruction *CtxI = nullptr) const {
9016 if (!getAnchorScope())
9017 return getWorstState(getBitWidth());
9018
9019 LazyValueInfo *LVI =
9020 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9021 *getAnchorScope());
9022
9023 if (!LVI || !CtxI)
9024 return getWorstState(getBitWidth());
9025 return LVI->getConstantRange(&getAssociatedValue(),
9026 const_cast<Instruction *>(CtxI),
9027 /*UndefAllowed*/ false);
9028 }
9029
9030 /// Return true if \p CtxI is valid for querying outside analyses.
9031 /// This basically makes sure we do not ask intra-procedural analysis
9032 /// about a context in the wrong function or a context that violates
9033 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9034 /// if the original context of this AA is OK or should be considered invalid.
9035 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9036 const Instruction *CtxI,
9037 bool AllowAACtxI) const {
9038 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9039 return false;
9040
9041 // Our context might be in a different function, neither intra-procedural
9042 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9043 if (!AA::isValidInScope(getAssociatedValue(), CtxI->getFunction()))
9044 return false;
9045
9046 // If the context is not dominated by the value there are paths to the
9047 // context that do not define the value. This cannot be handled by
9048 // LazyValueInfo so we need to bail.
9049 if (auto *I = dyn_cast<Instruction>(&getAssociatedValue())) {
9050 InformationCache &InfoCache = A.getInfoCache();
9051 const DominatorTree *DT =
9053 *I->getFunction());
9054 return DT && DT->dominates(I, CtxI);
9055 }
9056
9057 return true;
9058 }
9059
9060 /// See AAValueConstantRange::getKnownConstantRange(..).
9062 getKnownConstantRange(Attributor &A,
9063 const Instruction *CtxI = nullptr) const override {
9064 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9065 /* AllowAACtxI */ false))
9066 return getKnown();
9067
9068 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9069 ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
9070 return getKnown().intersectWith(SCEVR).intersectWith(LVIR);
9071 }
9072
9073 /// See AAValueConstantRange::getAssumedConstantRange(..).
9075 getAssumedConstantRange(Attributor &A,
9076 const Instruction *CtxI = nullptr) const override {
9077 // TODO: Make SCEV use Attributor assumption.
9078 // We may be able to bound a variable range via assumptions in
9079 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9080 // evolve to x^2 + x, then we can say that y is in [2, 12].
9081 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9082 /* AllowAACtxI */ false))
9083 return getAssumed();
9084
9085 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9086 ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
9087 return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);
9088 }
9089
9090 /// Helper function to create MDNode for range metadata.
9091 static MDNode *
9092 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9093 const ConstantRange &AssumedConstantRange) {
9094 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(ConstantInt::get(
9095 Ty, AssumedConstantRange.getLower())),
9096 ConstantAsMetadata::get(ConstantInt::get(
9097 Ty, AssumedConstantRange.getUpper()))};
9098 return MDNode::get(Ctx, LowAndHigh);
9099 }
9100
9101 /// Return true if \p Assumed is included in \p KnownRanges.
9102 static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
9103
9104 if (Assumed.isFullSet())
9105 return false;
9106
9107 if (!KnownRanges)
9108 return true;
9109
9110 // If multiple ranges are annotated in IR, we give up to annotate assumed
9111 // range for now.
9112
9113 // TODO: If there exists a known range which containts assumed range, we
9114 // can say assumed range is better.
9115 if (KnownRanges->getNumOperands() > 2)
9116 return false;
9117
9119 mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
9121 mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
9122
9123 ConstantRange Known(Lower->getValue(), Upper->getValue());
9124 return Known.contains(Assumed) && Known != Assumed;
9125 }
9126
9127 /// Helper function to set range metadata.
9128 static bool
9129 setRangeMetadataIfisBetterRange(Instruction *I,
9130 const ConstantRange &AssumedConstantRange) {
9131 auto *OldRangeMD = I->getMetadata(LLVMContext::MD_range);
9132 if (isBetterRange(AssumedConstantRange, OldRangeMD)) {
9133 if (!AssumedConstantRange.isEmptySet()) {
9134 I->setMetadata(LLVMContext::MD_range,
9135 getMDNodeForConstantRange(I->getType(), I->getContext(),
9136 AssumedConstantRange));
9137 return true;
9138 }
9139 }
9140 return false;
9141 }
9142
9143 /// See AbstractAttribute::manifest()
9144 ChangeStatus manifest(Attributor &A) override {
9145 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9146 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9147 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9148
9149 auto &V = getAssociatedValue();
9150 if (!AssumedConstantRange.isEmptySet() &&
9151 !AssumedConstantRange.isSingleElement()) {
9152 if (Instruction *I = dyn_cast<Instruction>(&V)) {
9153 assert(I == getCtxI() && "Should not annotate an instruction which is "
9154 "not the context instruction");
9155 if (isa<CallInst>(I) || isa<LoadInst>(I))
9156 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9157 Changed = ChangeStatus::CHANGED;
9158 }
9159 }
9160
9161 return Changed;
9162 }
9163};
9164
9165struct AAValueConstantRangeArgument final
9166 : AAArgumentFromCallSiteArguments<
9167 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9168 true /* BridgeCallBaseContext */> {
9169 using Base = AAArgumentFromCallSiteArguments<
9170 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9171 true /* BridgeCallBaseContext */>;
9172 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9173 : Base(IRP, A) {}
9174
9175 /// See AbstractAttribute::trackStatistics()
9176 void trackStatistics() const override {
9177 STATS_DECLTRACK_ARG_ATTR(value_range)
9178 }
9179};
9180
9181struct AAValueConstantRangeReturned
9182 : AAReturnedFromReturnedValues<AAValueConstantRange,
9183 AAValueConstantRangeImpl,
9184 AAValueConstantRangeImpl::StateType,
9185 /* PropogateCallBaseContext */ true> {
9186 using Base =
9187 AAReturnedFromReturnedValues<AAValueConstantRange,
9188 AAValueConstantRangeImpl,
9190 /* PropogateCallBaseContext */ true>;
9191 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9192 : Base(IRP, A) {}
9193
9194 /// See AbstractAttribute::initialize(...).
9195 void initialize(Attributor &A) override {
9196 if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
9197 indicatePessimisticFixpoint();
9198 }
9199
9200 /// See AbstractAttribute::trackStatistics()
9201 void trackStatistics() const override {
9202 STATS_DECLTRACK_FNRET_ATTR(value_range)
9203 }
9204};
9205
9206struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9207 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9208 : AAValueConstantRangeImpl(IRP, A) {}
9209
9210 /// See AbstractAttribute::initialize(...).
9211 void initialize(Attributor &A) override {
9212 AAValueConstantRangeImpl::initialize(A);
9213 if (isAtFixpoint())
9214 return;
9215
9216 Value &V = getAssociatedValue();
9217
9218 if (auto *C = dyn_cast<ConstantInt>(&V)) {
9219 unionAssumed(ConstantRange(C->getValue()));
9220 indicateOptimisticFixpoint();
9221 return;
9222 }
9223
9224 if (isa<UndefValue>(&V)) {
9225 // Collapse the undef state to 0.
9226 unionAssumed(ConstantRange(APInt(getBitWidth(), 0)));
9227 indicateOptimisticFixpoint();
9228 return;
9229 }
9230
9231 if (isa<CallBase>(&V))
9232 return;
9233
9234 if (isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<CastInst>(&V))
9235 return;
9236
9237 // If it is a load instruction with range metadata, use it.
9238 if (LoadInst *LI = dyn_cast<LoadInst>(&V))
9239 if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {
9240 intersectKnown(getConstantRangeFromMetadata(*RangeMD));
9241 return;
9242 }
9243
9244 // We can work with PHI and select instruction as we traverse their operands
9245 // during update.
9246 if (isa<SelectInst>(V) || isa<PHINode>(V))
9247 return;
9248
9249 // Otherwise we give up.
9250 indicatePessimisticFixpoint();
9251
9252 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9253 << getAssociatedValue() << "\n");
9254 }
9255
9256 bool calculateBinaryOperator(
9258 const Instruction *CtxI,
9260 Value *LHS = BinOp->getOperand(0);
9261 Value *RHS = BinOp->getOperand(1);
9262
9263 // Simplify the operands first.
9264 bool UsedAssumedInformation = false;
9265 const auto &SimplifiedLHS = A.getAssumedSimplified(
9266 IRPosition::value(*LHS, getCallBaseContext()), *this,
9267 UsedAssumedInformation, AA::Interprocedural);
9268 if (!SimplifiedLHS.has_value())
9269 return true;
9270 if (!*SimplifiedLHS)
9271 return false;
9272 LHS = *SimplifiedLHS;
9273
9274 const auto &SimplifiedRHS = A.getAssumedSimplified(
9275 IRPosition::value(*RHS, getCallBaseContext()), *this,
9276 UsedAssumedInformation, AA::Interprocedural);
9277 if (!SimplifiedRHS.has_value())
9278 return true;
9279 if (!*SimplifiedRHS)
9280 return false;
9281 RHS = *SimplifiedRHS;
9282
9283 // TODO: Allow non integers as well.
9284 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9285 return false;
9286
9287 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9288 *this, IRPosition::value(*LHS, getCallBaseContext()),
9289 DepClassTy::REQUIRED);
9290 if (!LHSAA)
9291 return false;
9292 QuerriedAAs.push_back(LHSAA);
9293 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9294
9295 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9296 *this, IRPosition::value(*RHS, getCallBaseContext()),
9297 DepClassTy::REQUIRED);
9298 if (!RHSAA)
9299 return false;
9300 QuerriedAAs.push_back(RHSAA);
9301 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9302
9303 auto AssumedRange = LHSAARange.binaryOp(BinOp->getOpcode(), RHSAARange);
9304
9305 T.unionAssumed(AssumedRange);
9306
9307 // TODO: Track a known state too.
9308
9309 return T.isValidState();
9310 }
9311
9312 bool calculateCastInst(
9314 const Instruction *CtxI,
9316 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9317 // TODO: Allow non integers as well.
9318 Value *OpV = CastI->getOperand(0);
9319
9320 // Simplify the operand first.
9321 bool UsedAssumedInformation = false;
9322 const auto &SimplifiedOpV = A.getAssumedSimplified(
9323 IRPosition::value(*OpV, getCallBaseContext()), *this,
9324 UsedAssumedInformation, AA::Interprocedural);
9325 if (!SimplifiedOpV.has_value())
9326 return true;
9327 if (!*SimplifiedOpV)
9328 return false;
9329 OpV = *SimplifiedOpV;
9330
9331 if (!OpV->getType()->isIntegerTy())
9332 return false;
9333
9334 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9335 *this, IRPosition::value(*OpV, getCallBaseContext()),
9336 DepClassTy::REQUIRED);
9337 if (!OpAA)
9338 return false;
9339 QuerriedAAs.push_back(OpAA);
9340 T.unionAssumed(OpAA->getAssumed().castOp(CastI->getOpcode(),
9341 getState().getBitWidth()));
9342 return T.isValidState();
9343 }
9344
9345 bool
9346 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9347 const Instruction *CtxI,
9349 Value *LHS = CmpI->getOperand(0);
9350 Value *RHS = CmpI->getOperand(1);
9351
9352 // Simplify the operands first.
9353 bool UsedAssumedInformation = false;
9354 const auto &SimplifiedLHS = A.getAssumedSimplified(
9355 IRPosition::value(*LHS, getCallBaseContext()), *this,
9356 UsedAssumedInformation, AA::Interprocedural);
9357 if (!SimplifiedLHS.has_value())
9358 return true;
9359 if (!*SimplifiedLHS)
9360 return false;
9361 LHS = *SimplifiedLHS;
9362
9363 const auto &SimplifiedRHS = A.getAssumedSimplified(
9364 IRPosition::value(*RHS, getCallBaseContext()), *this,
9365 UsedAssumedInformation, AA::Interprocedural);
9366 if (!SimplifiedRHS.has_value())
9367 return true;
9368 if (!*SimplifiedRHS)
9369 return false;
9370 RHS = *SimplifiedRHS;
9371
9372 // TODO: Allow non integers as well.
9373 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9374 return false;
9375
9376 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9377 *this, IRPosition::value(*LHS, getCallBaseContext()),
9378 DepClassTy::REQUIRED);
9379 if (!LHSAA)
9380 return false;
9381 QuerriedAAs.push_back(LHSAA);
9382 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9383 *this, IRPosition::value(*RHS, getCallBaseContext()),
9384 DepClassTy::REQUIRED);
9385 if (!RHSAA)
9386 return false;
9387 QuerriedAAs.push_back(RHSAA);
9388 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9389 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9390
9391 // If one of them is empty set, we can't decide.
9392 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9393 return true;
9394
9395 bool MustTrue = false, MustFalse = false;
9396
9397 auto AllowedRegion =
9399
9400 if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())
9401 MustFalse = true;
9402
9403 if (LHSAARange.icmp(CmpI->getPredicate(), RHSAARange))
9404 MustTrue = true;
9405
9406 assert((!MustTrue || !MustFalse) &&
9407 "Either MustTrue or MustFalse should be false!");
9408
9409 if (MustTrue)
9410 T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9411 else if (MustFalse)
9412 T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9413 else
9414 T.unionAssumed(ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9415
9416 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9417 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9418 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9419 << *RHSAA);
9420
9421 // TODO: Track a known state too.
9422 return T.isValidState();
9423 }
9424
9425 /// See AbstractAttribute::updateImpl(...).
9426 ChangeStatus updateImpl(Attributor &A) override {
9427
9429 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9430 Instruction *I = dyn_cast<Instruction>(&V);
9431 if (!I || isa<CallBase>(I)) {
9432
9433 // Simplify the operand first.
9434 bool UsedAssumedInformation = false;
9435 const auto &SimplifiedOpV = A.getAssumedSimplified(
9436 IRPosition::value(V, getCallBaseContext()), *this,
9437 UsedAssumedInformation, AA::Interprocedural);
9438 if (!SimplifiedOpV.has_value())
9439 return true;
9440 if (!*SimplifiedOpV)
9441 return false;
9442 Value *VPtr = *SimplifiedOpV;
9443
9444 // If the value is not instruction, we query AA to Attributor.
9445 const auto *AA = A.getAAFor<AAValueConstantRange>(
9446 *this, IRPosition::value(*VPtr, getCallBaseContext()),
9447 DepClassTy::REQUIRED);
9448
9449 // Clamp operator is not used to utilize a program point CtxI.
9450 if (AA)
9451 T.unionAssumed(AA->getAssumedConstantRange(A, CtxI));
9452 else
9453 return false;
9454
9455 return T.isValidState();
9456 }
9457
9459 if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {
9460 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9461 return false;
9462 } else if (auto *CmpI = dyn_cast<CmpInst>(I)) {
9463 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9464 return false;
9465 } else if (auto *CastI = dyn_cast<CastInst>(I)) {
9466 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9467 return false;
9468 } else {
9469 // Give up with other instructions.
9470 // TODO: Add other instructions
9471
9472 T.indicatePessimisticFixpoint();
9473 return false;
9474 }
9475
9476 // Catch circular reasoning in a pessimistic way for now.
9477 // TODO: Check how the range evolves and if we stripped anything, see also
9478 // AADereferenceable or AAAlign for similar situations.
9479 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9480 if (QueriedAA != this)
9481 continue;
9482 // If we are in a stady state we do not need to worry.
9483 if (T.getAssumed() == getState().getAssumed())
9484 continue;
9485 T.indicatePessimisticFixpoint();
9486 }
9487
9488 return T.isValidState();
9489 };
9490
9491 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9492 return indicatePessimisticFixpoint();
9493
9494 // Ensure that long def-use chains can't cause circular reasoning either by
9495 // introducing a cutoff below.
9496 if (clampStateAndIndicateChange(getState(), T) == ChangeStatus::UNCHANGED)
9497 return ChangeStatus::UNCHANGED;
9498 if (++NumChanges > MaxNumChanges) {
9499 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9500 << " but only " << MaxNumChanges
9501 << " are allowed to avoid cyclic reasoning.");
9502 return indicatePessimisticFixpoint();
9503 }
9504 return ChangeStatus::CHANGED;
9505 }
9506
9507 /// See AbstractAttribute::trackStatistics()
9508 void trackStatistics() const override {
9510 }
9511
9512 /// Tracker to bail after too many widening steps of the constant range.
9513 int NumChanges = 0;
9514
9515 /// Upper bound for the number of allowed changes (=widening steps) for the
9516 /// constant range before we give up.
9517 static constexpr int MaxNumChanges = 5;
9518};
9519
9520struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9521 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9522 : AAValueConstantRangeImpl(IRP, A) {}
9523
9524 /// See AbstractAttribute::initialize(...).
9525 ChangeStatus updateImpl(Attributor &A) override {
9526 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9527 "not be called");
9528 }
9529
9530 /// See AbstractAttribute::trackStatistics()
9531 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9532};
9533
9534struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9535 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9536 : AAValueConstantRangeFunction(IRP, A) {}
9537
9538 /// See AbstractAttribute::trackStatistics()
9539 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9540};
9541
9542struct AAValueConstantRangeCallSiteReturned
9543 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9544 AAValueConstantRangeImpl::StateType,
9545 /* IntroduceCallBaseContext */ true> {
9546 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9547 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9548 AAValueConstantRangeImpl::StateType,
9549 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9550
9551 /// See AbstractAttribute::initialize(...).
9552 void initialize(Attributor &A) override {
9553 // If it is a load instruction with range metadata, use the metadata.
9554 if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue()))
9555 if (auto *RangeMD = CI->getMetadata(LLVMContext::MD_range))
9556 intersectKnown(getConstantRangeFromMetadata(*RangeMD));
9557
9558 AAValueConstantRangeImpl::initialize(A);
9559 }
9560
9561 /// See AbstractAttribute::trackStatistics()
9562 void trackStatistics() const override {
9563 STATS_DECLTRACK_CSRET_ATTR(value_range)
9564 }
9565};
9566struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9567 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9568 : AAValueConstantRangeFloating(IRP, A) {}
9569
9570 /// See AbstractAttribute::manifest()
9571 ChangeStatus manifest(Attributor &A) override {
9572 return ChangeStatus::UNCHANGED;
9573 }
9574
9575 /// See AbstractAttribute::trackStatistics()
9576 void trackStatistics() const override {
9577 STATS_DECLTRACK_CSARG_ATTR(value_range)
9578 }
9579};
9580} // namespace
9581
9582/// ------------------ Potential Values Attribute -------------------------
9583
9584namespace {
9585struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9586 using StateType = PotentialConstantIntValuesState;
9587
9588 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9589 : AAPotentialConstantValues(IRP, A) {}
9590
9591 /// See AbstractAttribute::initialize(..).
9592 void initialize(Attributor &A) override {
9593 if (A.hasSimplificationCallback(getIRPosition()))
9594 indicatePessimisticFixpoint();
9595 else
9596 AAPotentialConstantValues::initialize(A);
9597 }
9598
9599 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9600 bool &ContainsUndef, bool ForSelf) {
9602 bool UsedAssumedInformation = false;
9603 if (!A.getAssumedSimplifiedValues(IRP, *this, Values, AA::Interprocedural,
9604 UsedAssumedInformation)) {
9605 // Avoid recursion when the caller is computing constant values for this
9606 // IRP itself.
9607 if (ForSelf)
9608 return false;
9609 if (!IRP.getAssociatedType()->isIntegerTy())
9610 return false;
9611 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9612 *this, IRP, DepClassTy::REQUIRED);
9613 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9614 return false;
9615 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9616 S = PotentialValuesAA->getState().getAssumedSet();
9617 return true;
9618 }
9619
9620 // Copy all the constant values, except UndefValue. ContainsUndef is true
9621 // iff Values contains only UndefValue instances. If there are other known
9622 // constants, then UndefValue is dropped.
9623 ContainsUndef = false;
9624 for (auto &It : Values) {
9625 if (isa<UndefValue>(It.getValue())) {
9626 ContainsUndef = true;
9627 continue;
9628 }
9629 auto *CI = dyn_cast<ConstantInt>(It.getValue());
9630 if (!CI)
9631 return false;
9632 S.insert(CI->getValue());
9633 }
9634 ContainsUndef &= S.empty();
9635
9636 return true;
9637 }
9638
9639 /// See AbstractAttribute::getAsStr().
9640 const std::string getAsStr(Attributor *A) const override {
9641 std::string Str;
9643 OS << getState();
9644 return OS.str();
9645 }
9646
9647 /// See AbstractAttribute::updateImpl(...).
9648 ChangeStatus updateImpl(Attributor &A) override {
9649 return indicatePessimisticFixpoint();
9650 }
9651};
9652
9653struct AAPotentialConstantValuesArgument final
9654 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9655 AAPotentialConstantValuesImpl,
9656 PotentialConstantIntValuesState> {
9657 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9658 AAPotentialConstantValuesImpl,
9660 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9661 : Base(IRP, A) {}
9662
9663 /// See AbstractAttribute::trackStatistics()
9664 void trackStatistics() const override {
9665 STATS_DECLTRACK_ARG_ATTR(potential_values)
9666 }
9667};
9668
9669struct AAPotentialConstantValuesReturned
9670 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9671 AAPotentialConstantValuesImpl> {
9672 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9673 AAPotentialConstantValuesImpl>;
9674 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9675 : Base(IRP, A) {}
9676
9677 void initialize(Attributor &A) override {
9678 if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
9679 indicatePessimisticFixpoint();
9680 Base::initialize(A);
9681 }
9682
9683 /// See AbstractAttribute::trackStatistics()
9684 void trackStatistics() const override {
9685 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9686 }
9687};
9688
9689struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9690 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9691 : AAPotentialConstantValuesImpl(IRP, A) {}
9692
9693 /// See AbstractAttribute::initialize(..).
9694 void initialize(Attributor &A) override {
9695 AAPotentialConstantValuesImpl::initialize(A);
9696 if (isAtFixpoint())
9697 return;
9698
9699 Value &V = getAssociatedValue();
9700
9701 if (auto *C = dyn_cast<ConstantInt>(&V)) {
9702 unionAssumed(C->getValue());
9703 indicateOptimisticFixpoint();
9704 return;
9705 }
9706
9707 if (isa<UndefValue>(&V)) {
9708 unionAssumedWithUndef();
9709 indicateOptimisticFixpoint();
9710 return;
9711 }
9712
9713 if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))
9714 return;
9715
9716 if (isa<SelectInst>(V) || isa<PHINode>(V) || isa<LoadInst>(V))
9717 return;
9718
9719 indicatePessimisticFixpoint();
9720
9721 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9722 << getAssociatedValue() << "\n");
9723 }
9724
9725 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9726 const APInt &RHS) {
9727 return ICmpInst::compare(LHS, RHS, ICI->getPredicate());
9728 }
9729
9730 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9731 uint32_t ResultBitWidth) {
9732 Instruction::CastOps CastOp = CI->getOpcode();
9733 switch (CastOp) {
9734 default:
9735 llvm_unreachable("unsupported or not integer cast");
9736 case Instruction::Trunc:
9737 return Src.trunc(ResultBitWidth);
9738 case Instruction::SExt:
9739 return Src.sext(ResultBitWidth);
9740 case Instruction::ZExt:
9741 return Src.zext(ResultBitWidth);
9742 case Instruction::BitCast:
9743 return Src;
9744 }
9745 }
9746
9747 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9748 const APInt &LHS, const APInt &RHS,
9749 bool &SkipOperation, bool &Unsupported) {
9750 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9751 // Unsupported is set to true when the binary operator is not supported.
9752 // SkipOperation is set to true when UB occur with the given operand pair
9753 // (LHS, RHS).
9754 // TODO: we should look at nsw and nuw keywords to handle operations
9755 // that create poison or undef value.
9756 switch (BinOpcode) {
9757 default:
9758 Unsupported = true;
9759 return LHS;
9760 case Instruction::Add:
9761 return LHS + RHS;
9762 case Instruction::Sub:
9763 return LHS - RHS;
9764 case Instruction::Mul:
9765 return LHS * RHS;
9766 case Instruction::UDiv:
9767 if (RHS.isZero()) {
9768 SkipOperation = true;
9769 return LHS;
9770 }
9771 return LHS.udiv(RHS);
9772 case Instruction::SDiv:
9773 if (RHS.isZero()) {
9774 SkipOperation = true;
9775 return LHS;
9776 }
9777 return LHS.sdiv(RHS);
9778 case Instruction::URem:
9779 if (RHS.isZero()) {
9780 SkipOperation = true;
9781 return LHS;
9782 }
9783 return LHS.urem(RHS);
9784 case Instruction::SRem:
9785 if (RHS.isZero()) {
9786 SkipOperation = true;
9787 return LHS;
9788 }
9789 return LHS.srem(RHS);
9790 case Instruction::Shl:
9791 return LHS.shl(RHS);
9792 case Instruction::LShr:
9793 return LHS.lshr(RHS);
9794 case Instruction::AShr:
9795 return LHS.ashr(RHS);
9796 case Instruction::And:
9797 return LHS & RHS;
9798 case Instruction::Or:
9799 return LHS | RHS;
9800 case Instruction::Xor:
9801 return LHS ^ RHS;
9802 }
9803 }
9804
9805 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9806 const APInt &LHS, const APInt &RHS) {
9807 bool SkipOperation = false;
9808 bool Unsupported = false;
9809 APInt Result =
9810 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9811 if (Unsupported)
9812 return false;
9813 // If SkipOperation is true, we can ignore this operand pair (L, R).
9814 if (!SkipOperation)
9815 unionAssumed(Result);
9816 return isValidState();
9817 }
9818
9819 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9820 auto AssumedBefore = getAssumed();
9821 Value *LHS = ICI->getOperand(0);
9822 Value *RHS = ICI->getOperand(1);
9823
9824 bool LHSContainsUndef = false, RHSContainsUndef = false;
9825 SetTy LHSAAPVS, RHSAAPVS;
9826 if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9827 LHSContainsUndef, /* ForSelf */ false) ||
9828 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9829 RHSContainsUndef, /* ForSelf */ false))
9830 return indicatePessimisticFixpoint();
9831
9832 // TODO: make use of undef flag to limit potential values aggressively.
9833 bool MaybeTrue = false, MaybeFalse = false;
9834 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
9835 if (LHSContainsUndef && RHSContainsUndef) {
9836 // The result of any comparison between undefs can be soundly replaced
9837 // with undef.
9838 unionAssumedWithUndef();
9839 } else if (LHSContainsUndef) {
9840 for (const APInt &R : RHSAAPVS) {
9841 bool CmpResult = calculateICmpInst(ICI, Zero, R);
9842 MaybeTrue |= CmpResult;
9843 MaybeFalse |= !CmpResult;
9844 if (MaybeTrue & MaybeFalse)
9845 return indicatePessimisticFixpoint();
9846 }
9847 } else if (RHSContainsUndef) {
9848 for (const APInt &L : LHSAAPVS) {
9849 bool CmpResult = calculateICmpInst(ICI, L, Zero);
9850 MaybeTrue |= CmpResult;
9851 MaybeFalse |= !CmpResult;
9852 if (MaybeTrue & MaybeFalse)
9853 return indicatePessimisticFixpoint();
9854 }
9855 } else {
9856 for (const APInt &L : LHSAAPVS) {
9857 for (const APInt &R : RHSAAPVS) {
9858 bool CmpResult = calculateICmpInst(ICI, L, R);
9859 MaybeTrue |= CmpResult;
9860 MaybeFalse |= !CmpResult;
9861 if (MaybeTrue & MaybeFalse)
9862 return indicatePessimisticFixpoint();
9863 }
9864 }
9865 }
9866 if (MaybeTrue)
9867 unionAssumed(APInt(/* numBits */ 1, /* val */ 1));
9868 if (MaybeFalse)
9869 unionAssumed(APInt(/* numBits */ 1, /* val */ 0));
9870 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9871 : ChangeStatus::CHANGED;
9872 }
9873
9874 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
9875 auto AssumedBefore = getAssumed();
9876 Value *LHS = SI->getTrueValue();
9877 Value *RHS = SI->getFalseValue();
9878
9879 bool UsedAssumedInformation = false;
9880 std::optional<Constant *> C = A.getAssumedConstant(
9881 *SI->getCondition(), *this, UsedAssumedInformation);
9882
9883 // Check if we only need one operand.
9884 bool OnlyLeft = false, OnlyRight = false;
9885 if (C && *C && (*C)->isOneValue())
9886 OnlyLeft = true;
9887 else if (C && *C && (*C)->isZeroValue())
9888 OnlyRight = true;
9889
9890 bool LHSContainsUndef = false, RHSContainsUndef = false;
9891 SetTy LHSAAPVS, RHSAAPVS;
9892 if (!OnlyRight &&
9893 !fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9894 LHSContainsUndef, /* ForSelf */ false))
9895 return indicatePessimisticFixpoint();
9896
9897 if (!OnlyLeft &&
9898 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9899 RHSContainsUndef, /* ForSelf */ false))
9900 return indicatePessimisticFixpoint();
9901
9902 if (OnlyLeft || OnlyRight) {
9903 // select (true/false), lhs, rhs
9904 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
9905 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
9906
9907 if (Undef)
9908 unionAssumedWithUndef();
9909 else {
9910 for (const auto &It : *OpAA)
9911 unionAssumed(It);
9912 }
9913
9914 } else if (LHSContainsUndef && RHSContainsUndef) {
9915 // select i1 *, undef , undef => undef
9916 unionAssumedWithUndef();
9917 } else {
9918 for (const auto &It : LHSAAPVS)
9919 unionAssumed(It);
9920 for (const auto &It : RHSAAPVS)
9921 unionAssumed(It);
9922 }
9923 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9924 : ChangeStatus::CHANGED;
9925 }
9926
9927 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
9928 auto AssumedBefore = getAssumed();
9929 if (!CI->isIntegerCast())
9930 return indicatePessimisticFixpoint();
9931 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
9932 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
9933 Value *Src = CI->getOperand(0);
9934
9935 bool SrcContainsUndef = false;
9936 SetTy SrcPVS;
9937 if (!fillSetWithConstantValues(A, IRPosition::value(*Src), SrcPVS,
9938 SrcContainsUndef, /* ForSelf */ false))
9939 return indicatePessimisticFixpoint();
9940
9941 if (SrcContainsUndef)
9942 unionAssumedWithUndef();
9943 else {
9944 for (const APInt &S : SrcPVS) {
9945 APInt T = calculateCastInst(CI, S, ResultBitWidth);
9946 unionAssumed(T);
9947 }
9948 }
9949 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9950 : ChangeStatus::CHANGED;
9951 }
9952
9953 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
9954 auto AssumedBefore = getAssumed();
9955 Value *LHS = BinOp->getOperand(0);
9956 Value *RHS = BinOp->getOperand(1);
9957
9958 bool LHSContainsUndef = false, RHSContainsUndef = false;
9959 SetTy LHSAAPVS, RHSAAPVS;
9960 if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9961 LHSContainsUndef, /* ForSelf */ false) ||
9962 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9963 RHSContainsUndef, /* ForSelf */ false))
9964 return indicatePessimisticFixpoint();
9965
9966 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
9967
9968 // TODO: make use of undef flag to limit potential values aggressively.
9969 if (LHSContainsUndef && RHSContainsUndef) {
9970 if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, Zero))
9971 return indicatePessimisticFixpoint();
9972 } else if (LHSContainsUndef) {
9973 for (const APInt &R : RHSAAPVS) {
9974 if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, R))
9975 return indicatePessimisticFixpoint();
9976 }
9977 } else if (RHSContainsUndef) {
9978 for (const APInt &L : LHSAAPVS) {
9979 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, Zero))
9980 return indicatePessimisticFixpoint();
9981 }
9982 } else {
9983 for (const APInt &L : LHSAAPVS) {
9984 for (const APInt &R : RHSAAPVS) {
9985 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))
9986 return indicatePessimisticFixpoint();
9987 }
9988 }
9989 }
9990 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9991 : ChangeStatus::CHANGED;
9992 }
9993
9994 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
9995 auto AssumedBefore = getAssumed();
9996 SetTy Incoming;
9997 bool ContainsUndef;
9998 if (!fillSetWithConstantValues(A, IRPosition::value(*Inst), Incoming,
9999 ContainsUndef, /* ForSelf */ true))
10000 return indicatePessimisticFixpoint();
10001 if (ContainsUndef) {
10002 unionAssumedWithUndef();
10003 } else {
10004 for (const auto &It : Incoming)
10005 unionAssumed(It);
10006 }
10007 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10008 : ChangeStatus::CHANGED;
10009 }
10010
10011 /// See AbstractAttribute::updateImpl(...).
10012 ChangeStatus updateImpl(Attributor &A) override {
10013 Value &V = getAssociatedValue();
10014 Instruction *I = dyn_cast<Instruction>(&V);
10015
10016 if (auto *ICI = dyn_cast<ICmpInst>(I))
10017 return updateWithICmpInst(A, ICI);
10018
10019 if (auto *SI = dyn_cast<SelectInst>(I))
10020 return updateWithSelectInst(A, SI);
10021
10022 if (auto *CI = dyn_cast<CastInst>(I))
10023 return updateWithCastInst(A, CI);
10024
10025 if (auto *BinOp = dyn_cast<BinaryOperator>(I))
10026 return updateWithBinaryOperator(A, BinOp);
10027
10028 if (isa<PHINode>(I) || isa<LoadInst>(I))
10029 return updateWithInstruction(A, I);
10030
10031 return indicatePessimisticFixpoint();
10032 }
10033
10034 /// See AbstractAttribute::trackStatistics()
10035 void trackStatistics() const override {
10036 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10037 }
10038};
10039
10040struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10041 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10042 : AAPotentialConstantValuesImpl(IRP, A) {}
10043
10044 /// See AbstractAttribute::initialize(...).
10045 ChangeStatus updateImpl(Attributor &A) override {
10047 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10048 "not be called");
10049 }
10050
10051 /// See AbstractAttribute::trackStatistics()
10052 void trackStatistics() const override {
10053 STATS_DECLTRACK_FN_ATTR(potential_values)
10054 }
10055};
10056
10057struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10058 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10059 : AAPotentialConstantValuesFunction(IRP, A) {}
10060
10061 /// See AbstractAttribute::trackStatistics()
10062 void trackStatistics() const override {
10063 STATS_DECLTRACK_CS_ATTR(potential_values)
10064 }
10065};
10066
10067struct AAPotentialConstantValuesCallSiteReturned
10068 : AACalleeToCallSite<AAPotentialConstantValues,
10069 AAPotentialConstantValuesImpl> {
10070 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10071 Attributor &A)
10072 : AACalleeToCallSite<AAPotentialConstantValues,
10073 AAPotentialConstantValuesImpl>(IRP, A) {}
10074
10075 /// See AbstractAttribute::trackStatistics()
10076 void trackStatistics() const override {
10077 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10078 }
10079};
10080
10081struct AAPotentialConstantValuesCallSiteArgument
10082 : AAPotentialConstantValuesFloating {
10083 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10084 Attributor &A)
10085 : AAPotentialConstantValuesFloating(IRP, A) {}
10086
10087 /// See AbstractAttribute::initialize(..).
10088 void initialize(Attributor &A) override {
10089 AAPotentialConstantValuesImpl::initialize(A);
10090 if (isAtFixpoint())
10091 return;
10092
10093 Value &V = getAssociatedValue();
10094
10095 if (auto *C = dyn_cast<ConstantInt>(&V)) {
10096 unionAssumed(C->getValue());
10097 indicateOptimisticFixpoint();
10098 return;
10099 }
10100
10101 if (isa<UndefValue>(&V)) {
10102 unionAssumedWithUndef();
10103 indicateOptimisticFixpoint();
10104 return;
10105 }
10106 }
10107
10108 /// See AbstractAttribute::updateImpl(...).
10109 ChangeStatus updateImpl(Attributor &A) override {
10110 Value &V = getAssociatedValue();
10111 auto AssumedBefore = getAssumed();
10112 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10113 *this, IRPosition::value(V), DepClassTy::REQUIRED);
10114 if (!AA)
10115 return indicatePessimisticFixpoint();
10116 const auto &S = AA->getAssumed();
10117 unionAssumed(S);
10118 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10119 : ChangeStatus::CHANGED;
10120 }
10121
10122 /// See AbstractAttribute::trackStatistics()
10123 void trackStatistics() const override {
10124 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10125 }
10126};
10127} // namespace
10128
10129/// ------------------------ NoUndef Attribute ---------------------------------
10131 Attribute::AttrKind ImpliedAttributeKind,
10132 bool IgnoreSubsumingPositions) {
10133 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10134 "Unexpected attribute kind");
10135 if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
10136 Attribute::NoUndef))
10137 return true;
10138
10139 Value &Val = IRP.getAssociatedValue();
10142 LLVMContext &Ctx = Val.getContext();
10143 A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));
10144 return true;
10145 }
10146
10147 return false;
10148}
10149
10150namespace {
10151struct AANoUndefImpl : AANoUndef {
10152 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10153
10154 /// See AbstractAttribute::initialize(...).
10155 void initialize(Attributor &A) override {
10156 Value &V = getAssociatedValue();
10157 if (isa<UndefValue>(V))
10158 indicatePessimisticFixpoint();
10159 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10160 }
10161
10162 /// See followUsesInMBEC
10163 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10164 AANoUndef::StateType &State) {
10165 const Value *UseV = U->get();
10166 const DominatorTree *DT = nullptr;
10167 AssumptionCache *AC = nullptr;
10168 InformationCache &InfoCache = A.getInfoCache();
10169 if (Function *F = getAnchorScope()) {
10172 }
10173 State.setKnown(isGuaranteedNotToBeUndefOrPoison(UseV, AC, I, DT));
10174 bool TrackUse = false;
10175 // Track use for instructions which must produce undef or poison bits when
10176 // at least one operand contains such bits.
10177 if (isa<CastInst>(*I) || isa<GetElementPtrInst>(*I))
10178 TrackUse = true;
10179 return TrackUse;
10180 }
10181
10182 /// See AbstractAttribute::getAsStr().
10183 const std::string getAsStr(Attributor *A) const override {
10184 return getAssumed() ? "noundef" : "may-undef-or-poison";
10185 }
10186
10187 ChangeStatus manifest(Attributor &A) override {
10188 // We don't manifest noundef attribute for dead positions because the
10189 // associated values with dead positions would be replaced with undef
10190 // values.
10191 bool UsedAssumedInformation = false;
10192 if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,
10193 UsedAssumedInformation))
10194 return ChangeStatus::UNCHANGED;
10195 // A position whose simplified value does not have any value is
10196 // considered to be dead. We don't manifest noundef in such positions for
10197 // the same reason above.
10198 if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,
10200 .has_value())
10201 return ChangeStatus::UNCHANGED;
10202 return AANoUndef::manifest(A);
10203 }
10204};
10205
10206struct AANoUndefFloating : public AANoUndefImpl {
10207 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10208 : AANoUndefImpl(IRP, A) {}
10209
10210 /// See AbstractAttribute::initialize(...).
10211 void initialize(Attributor &A) override {
10212 AANoUndefImpl::initialize(A);
10213 if (!getState().isAtFixpoint() && getAnchorScope() &&
10214 !getAnchorScope()->isDeclaration())
10215 if (Instruction *CtxI = getCtxI())
10216 followUsesInMBEC(*this, A, getState(), *CtxI);
10217 }
10218
10219 /// See AbstractAttribute::updateImpl(...).
10220 ChangeStatus updateImpl(Attributor &A) override {
10221 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10222 bool IsKnownNoUndef;
10223 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10224 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
10225 };
10226
10227 bool Stripped;
10228 bool UsedAssumedInformation = false;
10229 Value *AssociatedValue = &getAssociatedValue();
10231 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
10232 AA::AnyScope, UsedAssumedInformation))
10233 Stripped = false;
10234 else
10235 Stripped =
10236 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10237
10238 if (!Stripped) {
10239 // If we haven't stripped anything we might still be able to use a
10240 // different AA, but only if the IRP changes. Effectively when we
10241 // interpret this not as a call site value but as a floating/argument
10242 // value.
10243 const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
10244 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10245 return indicatePessimisticFixpoint();
10246 return ChangeStatus::UNCHANGED;
10247 }
10248
10249 for (const auto &VAC : Values)
10250 if (!VisitValueCB(IRPosition::value(*VAC.getValue())))
10251 return indicatePessimisticFixpoint();
10252
10253 return ChangeStatus::UNCHANGED;
10254 }
10255
10256 /// See AbstractAttribute::trackStatistics()
10257 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10258};
10259
10260struct AANoUndefReturned final
10261 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10262 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10263 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10264
10265 /// See AbstractAttribute::trackStatistics()
10266 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10267};
10268
10269struct AANoUndefArgument final
10270 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10271 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10272 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10273
10274 /// See AbstractAttribute::trackStatistics()
10275 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10276};
10277
10278struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10279 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10280 : AANoUndefFloating(IRP, A) {}
10281
10282 /// See AbstractAttribute::trackStatistics()
10283 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10284};
10285
10286struct AANoUndefCallSiteReturned final
10287 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10288 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10289 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10290
10291 /// See AbstractAttribute::trackStatistics()
10292 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10293};
10294
10295/// ------------------------ NoFPClass Attribute -------------------------------
10296
10297struct AANoFPClassImpl : AANoFPClass {
10298 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10299
10300 void initialize(Attributor &A) override {
10301 const IRPosition &IRP = getIRPosition();
10302
10303 Value &V = IRP.getAssociatedValue();
10304 if (isa<UndefValue>(V)) {
10305 indicateOptimisticFixpoint();
10306 return;
10307 }
10308
10310 A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);
10311 for (const auto &Attr : Attrs) {
10312 addKnownBits(Attr.getNoFPClass());
10313 }
10314
10315 const DataLayout &DL = A.getDataLayout();
10316 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10318 addKnownBits(~KnownFPClass.KnownFPClasses);
10319 }
10320
10321 if (Instruction *CtxI = getCtxI())
10322 followUsesInMBEC(*this, A, getState(), *CtxI);
10323 }
10324
10325 /// See followUsesInMBEC
10326 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10327 AANoFPClass::StateType &State) {
10328 const Value *UseV = U->get();
10329 const DominatorTree *DT = nullptr;
10330 AssumptionCache *AC = nullptr;
10331 const TargetLibraryInfo *TLI = nullptr;
10332 InformationCache &InfoCache = A.getInfoCache();
10333
10334 if (Function *F = getAnchorScope()) {
10337 TLI = InfoCache.getTargetLibraryInfoForFunction(*F);
10338 }
10339
10340 const DataLayout &DL = A.getDataLayout();
10341
10343 computeKnownFPClass(UseV, DL,
10344 /*InterestedClasses=*/fcAllFlags,
10345 /*Depth=*/0, TLI, AC, I, DT);
10346 State.addKnownBits(~KnownFPClass.KnownFPClasses);
10347
10348 if (auto *CI = dyn_cast<CallInst>(UseV)) {
10349 // Special case FP intrinsic with struct return type.
10350 switch (CI->getIntrinsicID()) {
10351 case Intrinsic::frexp:
10352 return true;
10354 // TODO: Could recognize math libcalls
10355 return false;
10356 default:
10357 break;
10358 }
10359 }
10360
10361 if (!UseV->getType()->isFPOrFPVectorTy())
10362 return false;
10363 return !isa<LoadInst, AtomicRMWInst>(UseV);
10364 }
10365
10366 const std::string getAsStr(Attributor *A) const override {
10367 std::string Result = "nofpclass";
10368 raw_string_ostream OS(Result);
10369 OS << getAssumedNoFPClass();
10370 return Result;
10371 }
10372
10373 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10374 SmallVectorImpl<Attribute> &Attrs) const override {
10375 Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));
10376 }
10377};
10378
10379struct AANoFPClassFloating : public AANoFPClassImpl {
10380 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10381 : AANoFPClassImpl(IRP, A) {}
10382
10383 /// See AbstractAttribute::updateImpl(...).
10384 ChangeStatus updateImpl(Attributor &A) override {
10386 bool UsedAssumedInformation = false;
10387 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
10388 AA::AnyScope, UsedAssumedInformation)) {
10389 Values.push_back({getAssociatedValue(), getCtxI()});
10390 }
10391
10392 StateType T;
10393 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10394 const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),
10395 DepClassTy::REQUIRED);
10396 if (!AA || this == AA) {
10397 T.indicatePessimisticFixpoint();
10398 } else {
10399 const AANoFPClass::StateType &S =
10400 static_cast<const AANoFPClass::StateType &>(AA->getState());
10401 T ^= S;
10402 }
10403 return T.isValidState();
10404 };
10405
10406 for (const auto &VAC : Values)
10407 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10408 return indicatePessimisticFixpoint();
10409
10410 return clampStateAndIndicateChange(getState(), T);
10411 }
10412
10413 /// See AbstractAttribute::trackStatistics()
10414 void trackStatistics() const override {
10416 }
10417};
10418
10419struct AANoFPClassReturned final
10420 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10421 AANoFPClassImpl::StateType, false, Attribute::None, false> {
10422 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10423 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10424 AANoFPClassImpl::StateType, false, Attribute::None, false>(
10425 IRP, A) {}
10426
10427 /// See AbstractAttribute::trackStatistics()
10428 void trackStatistics() const override {
10430 }
10431};
10432
10433struct AANoFPClassArgument final
10434 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10435 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10436 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10437
10438 /// See AbstractAttribute::trackStatistics()
10439 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10440};
10441
10442struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10443 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10444 : AANoFPClassFloating(IRP, A) {}
10445
10446 /// See AbstractAttribute::trackStatistics()
10447 void trackStatistics() const override {
10449 }
10450};
10451
10452struct AANoFPClassCallSiteReturned final
10453 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10454 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10455 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10456
10457 /// See AbstractAttribute::trackStatistics()
10458 void trackStatistics() const override {
10460 }
10461};
10462
10463struct AACallEdgesImpl : public AACallEdges {
10464 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10465
10466 const SetVector<Function *> &getOptimisticEdges() const override {
10467 return CalledFunctions;
10468 }
10469
10470 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10471
10472 bool hasNonAsmUnknownCallee() const override {
10473 return HasUnknownCalleeNonAsm;
10474 }
10475
10476 const std::string getAsStr(Attributor *A) const override {
10477 return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +
10478 std::to_string(CalledFunctions.size()) + "]";
10479 }
10480
10481 void trackStatistics() const override {}
10482
10483protected:
10484 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10485 if (CalledFunctions.insert(Fn)) {
10486 Change = ChangeStatus::CHANGED;
10487 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10488 << "\n");
10489 }
10490 }
10491
10492 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10493 if (!HasUnknownCallee)
10494 Change = ChangeStatus::CHANGED;
10495 if (NonAsm && !HasUnknownCalleeNonAsm)
10496 Change = ChangeStatus::CHANGED;
10497 HasUnknownCalleeNonAsm |= NonAsm;
10498 HasUnknownCallee = true;
10499 }
10500
10501private:
10502 /// Optimistic set of functions that might be called by this position.
10503 SetVector<Function *> CalledFunctions;
10504
10505 /// Is there any call with a unknown callee.
10506 bool HasUnknownCallee = false;
10507
10508 /// Is there any call with a unknown callee, excluding any inline asm.
10509 bool HasUnknownCalleeNonAsm = false;
10510};
10511
10512struct AACallEdgesCallSite : public AACallEdgesImpl {
10513 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10514 : AACallEdgesImpl(IRP, A) {}
10515 /// See AbstractAttribute::updateImpl(...).
10516 ChangeStatus updateImpl(Attributor &A) override {
10517 ChangeStatus Change = ChangeStatus::UNCHANGED;
10518
10519 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10520 if (Function *Fn = dyn_cast<Function>(&V)) {
10521 addCalledFunction(Fn, Change);
10522 } else {
10523 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10524 setHasUnknownCallee(true, Change);
10525 }
10526
10527 // Explore all values.
10528 return true;
10529 };
10530
10532 // Process any value that we might call.
10533 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10534 if (isa<Constant>(V)) {
10535 VisitValue(*V, CtxI);
10536 return;
10537 }
10538
10539 bool UsedAssumedInformation = false;
10540 Values.clear();
10541 if (!A.getAssumedSimplifiedValues(IRPosition::value(*V), *this, Values,
10542 AA::AnyScope, UsedAssumedInformation)) {
10543 Values.push_back({*V, CtxI});
10544 }
10545 for (auto &VAC : Values)
10546 VisitValue(*VAC.getValue(), VAC.getCtxI());
10547 };
10548
10549 CallBase *CB = cast<CallBase>(getCtxI());
10550
10551 if (auto *IA = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
10552 if (IA->hasSideEffects() &&
10553 !hasAssumption(*CB->getCaller(), "ompx_no_call_asm") &&
10554 !hasAssumption(*CB, "ompx_no_call_asm")) {
10555 setHasUnknownCallee(false, Change);
10556 }
10557 return Change;
10558 }
10559
10560 if (CB->isIndirectCall())
10561 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10562 *this, getIRPosition(), DepClassTy::OPTIONAL))
10563 if (IndirectCallAA->foreachCallee(
10564 [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10565 return Change;
10566
10567 // The most simple case.
10568 ProcessCalledOperand(CB->getCalledOperand(), CB);
10569
10570 // Process callback functions.
10571 SmallVector<const Use *, 4u> CallbackUses;
10572 AbstractCallSite::getCallbackUses(*CB, CallbackUses);
10573 for (const Use *U : CallbackUses)
10574 ProcessCalledOperand(U->get(), CB);
10575
10576 return Change;
10577 }
10578};
10579
10580struct AACallEdgesFunction : public AACallEdgesImpl {
10581 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10582 : AACallEdgesImpl(IRP, A) {}
10583
10584 /// See AbstractAttribute::updateImpl(...).
10585 ChangeStatus updateImpl(Attributor &A) override {
10586 ChangeStatus Change = ChangeStatus::UNCHANGED;
10587
10588 auto ProcessCallInst = [&](Instruction &Inst) {
10589 CallBase &CB = cast<CallBase>(Inst);
10590
10591 auto *CBEdges = A.getAAFor<AACallEdges>(
10592 *this, IRPosition::callsite_function(CB), DepClassTy::REQUIRED);
10593 if (!CBEdges)
10594 return false;
10595 if (CBEdges->hasNonAsmUnknownCallee())
10596 setHasUnknownCallee(true, Change);
10597 if (CBEdges->hasUnknownCallee())
10598 setHasUnknownCallee(false, Change);
10599
10600 for (Function *F : CBEdges->getOptimisticEdges())
10601 addCalledFunction(F, Change);
10602
10603 return true;
10604 };
10605
10606 // Visit all callable instructions.
10607 bool UsedAssumedInformation = false;
10608 if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,
10609 UsedAssumedInformation,
10610 /* CheckBBLivenessOnly */ true)) {
10611 // If we haven't looked at all call like instructions, assume that there
10612 // are unknown callees.
10613 setHasUnknownCallee(true, Change);
10614 }
10615
10616 return Change;
10617 }
10618};
10619
10620/// -------------------AAInterFnReachability Attribute--------------------------
10621
10622struct AAInterFnReachabilityFunction
10623 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10624 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10625 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10626 : Base(IRP, A) {}
10627
10628 bool instructionCanReach(
10629 Attributor &A, const Instruction &From, const Function &To,
10630 const AA::InstExclusionSetTy *ExclusionSet) const override {
10631 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10632 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10633
10634 RQITy StackRQI(A, From, To, ExclusionSet, false);
10635 typename RQITy::Reachable Result;
10636 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10637 return NonConstThis->isReachableImpl(A, StackRQI,
10638 /*IsTemporaryRQI=*/true);
10639 return Result == RQITy::Reachable::Yes;
10640 }
10641
10642 bool isReachableImpl(Attributor &A, RQITy &RQI,
10643 bool IsTemporaryRQI) override {
10644 const Instruction *EntryI =
10645 &RQI.From->getFunction()->getEntryBlock().front();
10646 if (EntryI != RQI.From &&
10647 !instructionCanReach(A, *EntryI, *RQI.To, nullptr))
10648 return rememberResult(A, RQITy::Reachable::No, RQI, false,
10649 IsTemporaryRQI);
10650
10651 auto CheckReachableCallBase = [&](CallBase *CB) {
10652 auto *CBEdges = A.getAAFor<AACallEdges>(
10653 *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
10654 if (!CBEdges || !CBEdges->getState().isValidState())
10655 return false;
10656 // TODO Check To backwards in this case.
10657 if (CBEdges->hasUnknownCallee())
10658 return false;
10659
10660 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10661 if (Fn == RQI.To)
10662 return false;
10663
10664 if (Fn->isDeclaration()) {
10665 if (Fn->hasFnAttribute(Attribute::NoCallback))
10666 continue;
10667 // TODO Check To backwards in this case.
10668 return false;
10669 }
10670
10671 if (Fn == getAnchorScope()) {
10672 if (EntryI == RQI.From)
10673 continue;
10674 return false;
10675 }
10676
10677 const AAInterFnReachability *InterFnReachability =
10678 A.getAAFor<AAInterFnReachability>(*this, IRPosition::function(*Fn),
10679 DepClassTy::OPTIONAL);
10680
10681 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10682 if (!InterFnReachability ||
10683 InterFnReachability->instructionCanReach(A, FnFirstInst, *RQI.To,
10684 RQI.ExclusionSet))
10685 return false;
10686 }
10687 return true;
10688 };
10689
10690 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10691 *this, IRPosition::function(*RQI.From->getFunction()),
10692 DepClassTy::OPTIONAL);
10693
10694 // Determine call like instructions that we can reach from the inst.
10695 auto CheckCallBase = [&](Instruction &CBInst) {
10696 // There are usually less nodes in the call graph, check inter function
10697 // reachability first.
10698 if (CheckReachableCallBase(cast<CallBase>(&CBInst)))
10699 return true;
10700 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10701 A, *RQI.From, CBInst, RQI.ExclusionSet);
10702 };
10703
10704 bool UsedExclusionSet = /* conservative */ true;
10705 bool UsedAssumedInformation = false;
10706 if (!A.checkForAllCallLikeInstructions(CheckCallBase, *this,
10707 UsedAssumedInformation,
10708 /* CheckBBLivenessOnly */ true))
10709 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10710 IsTemporaryRQI);
10711
10712 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
10713 IsTemporaryRQI);
10714 }
10715
10716 void trackStatistics() const override {}
10717};
10718} // namespace
10719
10720template <typename AAType>
10721static std::optional<Constant *>
10723 const IRPosition &IRP, Type &Ty) {
10724 if (!Ty.isIntegerTy())
10725 return nullptr;
10726
10727 // This will also pass the call base context.
10728 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10729 if (!AA)
10730 return nullptr;
10731
10732 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10733
10734 if (!COpt.has_value()) {
10735 A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
10736 return std::nullopt;
10737 }
10738 if (auto *C = *COpt) {
10739 A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
10740 return C;
10741 }
10742 return nullptr;
10743}
10744
10746 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10748 Type &Ty = *IRP.getAssociatedType();
10749 std::optional<Value *> V;
10750 for (auto &It : Values) {
10751 V = AA::combineOptionalValuesInAAValueLatice(V, It.getValue(), &Ty);
10752 if (V.has_value() && !*V)
10753 break;
10754 }
10755 if (!V.has_value())
10756 return UndefValue::get(&Ty);
10757 return *V;
10758}
10759
10760namespace {
10761struct AAPotentialValuesImpl : AAPotentialValues {
10762 using StateType = PotentialLLVMValuesState;
10763
10764 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10765 : AAPotentialValues(IRP, A) {}
10766
10767 /// See AbstractAttribute::initialize(..).
10768 void initialize(Attributor &A) override {
10769 if (A.hasSimplificationCallback(getIRPosition())) {
10770 indicatePessimisticFixpoint();
10771 return;
10772 }
10773 Value *Stripped = getAssociatedValue().stripPointerCasts();
10774 auto *CE = dyn_cast<ConstantExpr>(Stripped);
10775 if (isa<Constant>(Stripped) &&
10776 (!CE || CE->getOpcode() != Instruction::ICmp)) {
10777 addValue(A, getState(), *Stripped, getCtxI(), AA::AnyScope,
10778 getAnchorScope());
10779 indicateOptimisticFixpoint();
10780 return;
10781 }
10782 AAPotentialValues::initialize(A);
10783 }
10784
10785 /// See AbstractAttribute::getAsStr().
10786 const std::string getAsStr(Attributor *A) const override {
10787 std::string Str;
10789 OS << getState();
10790 return OS.str();
10791 }
10792
10793 template <typename AAType>
10794 static std::optional<Value *> askOtherAA(Attributor &A,
10795 const AbstractAttribute &AA,
10796 const IRPosition &IRP, Type &Ty) {
10797 if (isa<Constant>(IRP.getAssociatedValue()))
10798 return &IRP.getAssociatedValue();
10799 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10800 if (!C)
10801 return std::nullopt;
10802 if (*C)
10803 if (auto *CC = AA::getWithType(**C, Ty))
10804 return CC;
10805 return nullptr;
10806 }
10807
10808 virtual void addValue(Attributor &A, StateType &State, Value &V,
10809 const Instruction *CtxI, AA::ValueScope S,
10810 Function *AnchorScope) const {
10811
10812 IRPosition ValIRP = IRPosition::value(V);
10813 if (auto *CB = dyn_cast_or_null<CallBase>(CtxI)) {
10814 for (const auto &U : CB->args()) {
10815 if (U.get() != &V)
10816 continue;
10817 ValIRP = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
10818 break;
10819 }
10820 }
10821
10822 Value *VPtr = &V;
10823 if (ValIRP.getAssociatedType()->isIntegerTy()) {
10824 Type &Ty = *getAssociatedType();
10825 std::optional<Value *> SimpleV =
10826 askOtherAA<AAValueConstantRange>(A, *this, ValIRP, Ty);
10827 if (SimpleV.has_value() && !*SimpleV) {
10828 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10829 *this, ValIRP, DepClassTy::OPTIONAL);
10830 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10831 for (const auto &It : PotentialConstantsAA->getAssumedSet())
10832 State.unionAssumed({{*ConstantInt::get(&Ty, It), nullptr}, S});
10833 if (PotentialConstantsAA->undefIsContained())
10834 State.unionAssumed({{*UndefValue::get(&Ty), nullptr}, S});
10835 return;
10836 }
10837 }
10838 if (!SimpleV.has_value())
10839 return;
10840
10841 if (*SimpleV)
10842 VPtr = *SimpleV;
10843 }
10844
10845 if (isa<ConstantInt>(VPtr))
10846 CtxI = nullptr;
10847 if (!AA::isValidInScope(*VPtr, AnchorScope))
10849
10850 State.unionAssumed({{*VPtr, CtxI}, S});
10851 }
10852
10853 /// Helper struct to tie a value+context pair together with the scope for
10854 /// which this is the simplified version.
10855 struct ItemInfo {
10858
10859 bool operator==(const ItemInfo &II) const {
10860 return II.I == I && II.S == S;
10861 };
10862 bool operator<(const ItemInfo &II) const {
10863 if (I == II.I)
10864 return S < II.S;
10865 return I < II.I;
10866 };
10867 };
10868
10869 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
10871 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
10872 if (!(CS & S))
10873 continue;
10874
10875 bool UsedAssumedInformation = false;
10877 if (!A.getAssumedSimplifiedValues(IRP, this, Values, CS,
10878 UsedAssumedInformation))
10879 return false;
10880
10881 for (auto &It : Values)
10882 ValueScopeMap[It] += CS;
10883 }
10884 for (auto &It : ValueScopeMap)
10885 addValue(A, getState(), *It.first.getValue(), It.first.getCtxI(),
10886 AA::ValueScope(It.second), getAnchorScope());
10887
10888 return true;
10889 }
10890
10891 void giveUpOnIntraprocedural(Attributor &A) {
10892 auto NewS = StateType::getBestState(getState());
10893 for (const auto &It : getAssumedSet()) {
10894 if (It.second == AA::Intraprocedural)
10895 continue;
10896 addValue(A, NewS, *It.first.getValue(), It.first.getCtxI(),
10897 AA::Interprocedural, getAnchorScope());
10898 }
10899 assert(!undefIsContained() && "Undef should be an explicit value!");
10900 addValue(A, NewS, getAssociatedValue(), getCtxI(), AA::Intraprocedural,
10901 getAnchorScope());
10902 getState() = NewS;
10903 }
10904
10905 /// See AbstractState::indicatePessimisticFixpoint(...).
10906 ChangeStatus indicatePessimisticFixpoint() override {
10907 getState() = StateType::getBestState(getState());
10908 getState().unionAssumed({{getAssociatedValue(), getCtxI()}, AA::AnyScope});
10910 return ChangeStatus::CHANGED;
10911 }
10912
10913 /// See AbstractAttribute::updateImpl(...).
10914 ChangeStatus updateImpl(Attributor &A) override {
10915 return indicatePessimisticFixpoint();
10916 }
10917
10918 /// See AbstractAttribute::manifest(...).
10919 ChangeStatus manifest(Attributor &A) override {
10922 Values.clear();
10923 if (!getAssumedSimplifiedValues(A, Values, S))
10924 continue;
10925 Value &OldV = getAssociatedValue();
10926 if (isa<UndefValue>(OldV))
10927 continue;
10928 Value *NewV = getSingleValue(A, *this, getIRPosition(), Values);
10929 if (!NewV || NewV == &OldV)
10930 continue;
10931 if (getCtxI() &&
10932 !AA::isValidAtPosition({*NewV, *getCtxI()}, A.getInfoCache()))
10933 continue;
10934 if (A.changeAfterManifest(getIRPosition(), *NewV))
10935 return ChangeStatus::CHANGED;
10936 }
10938 }
10939
10940 bool getAssumedSimplifiedValues(
10942 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
10943 if (!isValidState())
10944 return false;
10945 bool UsedAssumedInformation = false;
10946 for (const auto &It : getAssumedSet())
10947 if (It.second & S) {
10948 if (RecurseForSelectAndPHI && (isa<PHINode>(It.first.getValue()) ||
10949 isa<SelectInst>(It.first.getValue()))) {
10950 if (A.getAssumedSimplifiedValues(
10951 IRPosition::inst(*cast<Instruction>(It.first.getValue())),
10952 this, Values, S, UsedAssumedInformation))
10953 continue;
10954 }
10955 Values.push_back(It.first);
10956 }
10957 assert(!undefIsContained() && "Undef should be an explicit value!");
10958 return true;
10959 }
10960};
10961
10962struct AAPotentialValuesFloating : AAPotentialValuesImpl {
10963 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
10964 : AAPotentialValuesImpl(IRP, A) {}
10965
10966 /// See AbstractAttribute::updateImpl(...).
10967 ChangeStatus updateImpl(Attributor &A) override {
10968 auto AssumedBefore = getAssumed();
10969
10970 genericValueTraversal(A, &getAssociatedValue());
10971
10972 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
10973 : ChangeStatus::CHANGED;
10974 }
10975
10976 /// Helper struct to remember which AAIsDead instances we actually used.
10977 struct LivenessInfo {
10978 const AAIsDead *LivenessAA = nullptr;
10979 bool AnyDead = false;
10980 };
10981
10982 /// Check if \p Cmp is a comparison we can simplify.
10983 ///
10984 /// We handle multiple cases, one in which at least one operand is an
10985 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
10986 /// operand. Return true if successful, in that case Worklist will be updated.
10987 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
10988 CmpInst::Predicate Pred, ItemInfo II,
10989 SmallVectorImpl<ItemInfo> &Worklist) {
10990
10991 // Simplify the operands first.
10992 bool UsedAssumedInformation = false;
10993 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
10994 auto GetSimplifiedValues = [&](Value &V,
10996 if (!A.getAssumedSimplifiedValues(
10997 IRPosition::value(V, getCallBaseContext()), this, Values,
10998 AA::Intraprocedural, UsedAssumedInformation)) {
10999 Values.clear();
11000 Values.push_back(AA::ValueAndContext{V, II.I.getCtxI()});
11001 }
11002 return Values.empty();
11003 };
11004 if (GetSimplifiedValues(*LHS, LHSValues))
11005 return true;
11006 if (GetSimplifiedValues(*RHS, RHSValues))
11007 return true;
11008
11009 LLVMContext &Ctx = LHS->getContext();
11010
11011 InformationCache &InfoCache = A.getInfoCache();
11012 Instruction *CmpI = dyn_cast<Instruction>(&Cmp);
11013 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11014 const auto *DT =
11016 : nullptr;
11017 const auto *TLI =
11018 F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
11019 auto *AC =
11021 : nullptr;
11022
11023 const DataLayout &DL = A.getDataLayout();
11024 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11025
11026 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11027 if (isa<UndefValue>(LHSV) || isa<UndefValue>(RHSV)) {
11028 addValue(A, getState(), *UndefValue::get(Cmp.getType()),
11029 /* CtxI */ nullptr, II.S, getAnchorScope());
11030 return true;
11031 }
11032
11033 // Handle the trivial case first in which we don't even need to think
11034 // about null or non-null.
11035 if (&LHSV == &RHSV &&
11037 Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),
11039 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11040 getAnchorScope());
11041 return true;
11042 }
11043
11044 auto *TypedLHS = AA::getWithType(LHSV, *LHS->getType());
11045 auto *TypedRHS = AA::getWithType(RHSV, *RHS->getType());
11046 if (TypedLHS && TypedRHS) {
11047 Value *NewV = simplifyCmpInst(Pred, TypedLHS, TypedRHS, Q);
11048 if (NewV && NewV != &Cmp) {
11049 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11050 getAnchorScope());
11051 return true;
11052 }
11053 }
11054
11055 // From now on we only handle equalities (==, !=).
11056 if (!CmpInst::isEquality(Pred))
11057 return false;
11058
11059 bool LHSIsNull = isa<ConstantPointerNull>(LHSV);
11060 bool RHSIsNull = isa<ConstantPointerNull>(RHSV);
11061 if (!LHSIsNull && !RHSIsNull)
11062 return false;
11063
11064 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11065 // non-nullptr operand and if we assume it's non-null we can conclude the
11066 // result of the comparison.
11067 assert((LHSIsNull || RHSIsNull) &&
11068 "Expected nullptr versus non-nullptr comparison at this point");
11069
11070 // The index is the operand that we assume is not null.
11071 unsigned PtrIdx = LHSIsNull;
11072 bool IsKnownNonNull;
11073 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11074 A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),
11075 DepClassTy::REQUIRED, IsKnownNonNull);
11076 if (!IsAssumedNonNull)
11077 return false;
11078
11079 // The new value depends on the predicate, true for != and false for ==.
11080 Constant *NewV =
11081 ConstantInt::get(Type::getInt1Ty(Ctx), Pred == CmpInst::ICMP_NE);
11082 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11083 getAnchorScope());
11084 return true;
11085 };
11086
11087 for (auto &LHSValue : LHSValues)
11088 for (auto &RHSValue : RHSValues)
11089 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11090 return false;
11091 return true;
11092 }
11093
11094 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11095 SmallVectorImpl<ItemInfo> &Worklist) {
11096 const Instruction *CtxI = II.I.getCtxI();
11097 bool UsedAssumedInformation = false;
11098
11099 std::optional<Constant *> C =
11100 A.getAssumedConstant(*SI.getCondition(), *this, UsedAssumedInformation);
11101 bool NoValueYet = !C.has_value();
11102 if (NoValueYet || isa_and_nonnull<UndefValue>(*C))
11103 return true;
11104 if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) {
11105 if (CI->isZero())
11106 Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
11107 else
11108 Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
11109 } else if (&SI == &getAssociatedValue()) {
11110 // We could not simplify the condition, assume both values.
11111 Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
11112 Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
11113 } else {
11114 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11115 IRPosition::inst(SI), *this, UsedAssumedInformation, II.S);
11116 if (!SimpleV.has_value())
11117 return true;
11118 if (*SimpleV) {
11119 addValue(A, getState(), **SimpleV, CtxI, II.S, getAnchorScope());
11120 return true;
11121 }
11122 return false;
11123 }
11124 return true;
11125 }
11126
11127 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11128 SmallVectorImpl<ItemInfo> &Worklist) {
11129 SmallSetVector<Value *, 4> PotentialCopies;
11130 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11131 bool UsedAssumedInformation = false;
11132 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialCopies,
11133 PotentialValueOrigins, *this,
11134 UsedAssumedInformation,
11135 /* OnlyExact */ true)) {
11136 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11137 "loaded values for load instruction "
11138 << LI << "\n");
11139 return false;
11140 }
11141
11142 // Do not simplify loads that are only used in llvm.assume if we cannot also
11143 // remove all stores that may feed into the load. The reason is that the
11144 // assume is probably worth something as long as the stores are around.
11145 InformationCache &InfoCache = A.getInfoCache();
11146 if (InfoCache.isOnlyUsedByAssume(LI)) {
11147 if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) {
11148 if (!I || isa<AssumeInst>(I))
11149 return true;
11150 if (auto *SI = dyn_cast<StoreInst>(I))
11151 return A.isAssumedDead(SI->getOperandUse(0), this,
11152 /* LivenessAA */ nullptr,
11153 UsedAssumedInformation,
11154 /* CheckBBLivenessOnly */ false);
11155 return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr,
11156 UsedAssumedInformation,
11157 /* CheckBBLivenessOnly */ false);
11158 })) {
11159 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11160 "and we cannot delete all the stores: "
11161 << LI << "\n");
11162 return false;
11163 }
11164 }
11165
11166 // Values have to be dynamically unique or we loose the fact that a
11167 // single llvm::Value might represent two runtime values (e.g.,
11168 // stack locations in different recursive calls).
11169 const Instruction *CtxI = II.I.getCtxI();
11170 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11171 bool AllLocal = ScopeIsLocal;
11172 bool DynamicallyUnique = llvm::all_of(PotentialCopies, [&](Value *PC) {
11173 AllLocal &= AA::isValidInScope(*PC, getAnchorScope());
11174 return AA::isDynamicallyUnique(A, *this, *PC);
11175 });
11176 if (!DynamicallyUnique) {
11177 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11178 "values are dynamically unique: "
11179 << LI << "\n");
11180 return false;
11181 }
11182
11183 for (auto *PotentialCopy : PotentialCopies) {
11184 if (AllLocal) {
11185 Worklist.push_back({{*PotentialCopy, CtxI}, II.S});
11186 } else {
11187 Worklist.push_back({{*PotentialCopy, CtxI}, AA::Interprocedural});
11188 }
11189 }
11190 if (!AllLocal && ScopeIsLocal)
11191 addValue(A, getState(), LI, CtxI, AA::Intraprocedural, getAnchorScope());
11192 return true;
11193 }
11194
11195 bool handlePHINode(
11196 Attributor &A, PHINode &PHI, ItemInfo II,
11197 SmallVectorImpl<ItemInfo> &Worklist,
11199 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11200 LivenessInfo &LI = LivenessAAs[&F];
11201 if (!LI.LivenessAA)
11202 LI.LivenessAA = A.getAAFor<AAIsDead>(*this, IRPosition::function(F),
11203 DepClassTy::NONE);
11204 return LI;
11205 };
11206
11207 if (&PHI == &getAssociatedValue()) {
11208 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11209 const auto *CI =
11210 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11211 *PHI.getFunction());
11212
11213 Cycle *C = nullptr;
11214 bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
11215 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11216 BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
11217 if (LI.LivenessAA &&
11218 LI.LivenessAA->isEdgeDead(IncomingBB, PHI.getParent())) {
11219 LI.AnyDead = true;
11220 continue;
11221 }
11222 Value *V = PHI.getIncomingValue(u);
11223 if (V == &PHI)
11224 continue;
11225
11226 // If the incoming value is not the PHI but an instruction in the same
11227 // cycle we might have multiple versions of it flying around.
11228 if (CyclePHI && isa<Instruction>(V) &&
11229 (!C || C->contains(cast<Instruction>(V)->getParent())))
11230 return false;
11231
11232 Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
11233 }
11234 return true;
11235 }
11236
11237 bool UsedAssumedInformation = false;
11238 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11239 IRPosition::inst(PHI), *this, UsedAssumedInformation, II.S);
11240 if (!SimpleV.has_value())
11241 return true;
11242 if (!(*SimpleV))
11243 return false;
11244 addValue(A, getState(), **SimpleV, &PHI, II.S, getAnchorScope());
11245 return true;
11246 }
11247
11248 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11249 /// simplify any operand of the instruction \p I. Return true if successful,
11250 /// in that case Worklist will be updated.
11251 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11252 SmallVectorImpl<ItemInfo> &Worklist) {
11253 bool SomeSimplified = false;
11254 bool UsedAssumedInformation = false;
11255
11256 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11257 int Idx = 0;
11258 for (Value *Op : I.operands()) {
11259 const auto &SimplifiedOp = A.getAssumedSimplified(
11260 IRPosition::value(*Op, getCallBaseContext()), *this,
11261 UsedAssumedInformation, AA::Intraprocedural);
11262 // If we are not sure about any operand we are not sure about the entire
11263 // instruction, we'll wait.
11264 if (!SimplifiedOp.has_value())
11265 return true;
11266
11267 if (*SimplifiedOp)
11268 NewOps[Idx] = *SimplifiedOp;
11269 else
11270 NewOps[Idx] = Op;
11271
11272 SomeSimplified |= (NewOps[Idx] != Op);
11273 ++Idx;
11274 }
11275
11276 // We won't bother with the InstSimplify interface if we didn't simplify any
11277 // operand ourselves.
11278 if (!SomeSimplified)
11279 return false;
11280
11281 InformationCache &InfoCache = A.getInfoCache();
11282 Function *F = I.getFunction();
11283 const auto *DT =
11285 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
11286 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
11287
11288 const DataLayout &DL = I.getModule()->getDataLayout();
11289 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11290 Value *NewV = simplifyInstructionWithOperands(&I, NewOps, Q);
11291 if (!NewV || NewV == &I)
11292 return false;
11293
11294 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11295 << *NewV << "\n");
11296 Worklist.push_back({{*NewV, II.I.getCtxI()}, II.S});
11297 return true;
11298 }
11299
11301 Attributor &A, Instruction &I, ItemInfo II,
11302 SmallVectorImpl<ItemInfo> &Worklist,
11304 if (auto *CI = dyn_cast<CmpInst>(&I))
11305 return handleCmp(A, *CI, CI->getOperand(0), CI->getOperand(1),
11306 CI->getPredicate(), II, Worklist);
11307
11308 switch (I.getOpcode()) {
11309 case Instruction::Select:
11310 return handleSelectInst(A, cast<SelectInst>(I), II, Worklist);
11311 case Instruction::PHI:
11312 return handlePHINode(A, cast<PHINode>(I), II, Worklist, LivenessAAs);
11313 case Instruction::Load:
11314 return handleLoadInst(A, cast<LoadInst>(I), II, Worklist);
11315 default:
11316 return handleGenericInst(A, I, II, Worklist);
11317 };
11318 return false;
11319 }
11320
11321 void genericValueTraversal(Attributor &A, Value *InitialV) {
11323
11324 SmallSet<ItemInfo, 16> Visited;
11326 Worklist.push_back({{*InitialV, getCtxI()}, AA::AnyScope});
11327
11328 int Iteration = 0;
11329 do {
11330 ItemInfo II = Worklist.pop_back_val();
11331 Value *V = II.I.getValue();
11332 assert(V);
11333 const Instruction *CtxI = II.I.getCtxI();
11334 AA::ValueScope S = II.S;
11335
11336 // Check if we should process the current value. To prevent endless
11337 // recursion keep a record of the values we followed!
11338 if (!Visited.insert(II).second)
11339 continue;
11340
11341 // Make sure we limit the compile time for complex expressions.
11342 if (Iteration++ >= MaxPotentialValuesIterations) {
11343 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11344 << Iteration << "!\n");
11345 addValue(A, getState(), *V, CtxI, S, getAnchorScope());
11346 continue;
11347 }
11348
11349 // Explicitly look through calls with a "returned" attribute if we do
11350 // not have a pointer as stripPointerCasts only works on them.
11351 Value *NewV = nullptr;
11352 if (V->getType()->isPointerTy()) {
11353 NewV = AA::getWithType(*V->stripPointerCasts(), *V->getType());
11354 } else {
11355 if (auto *CB = dyn_cast<CallBase>(V))
11356 if (auto *Callee =
11357 dyn_cast_if_present<Function>(CB->getCalledOperand())) {
11358 for (Argument &Arg : Callee->args())
11359 if (Arg.hasReturnedAttr()) {
11360 NewV = CB->getArgOperand(Arg.getArgNo());
11361 break;
11362 }
11363 }
11364 }
11365 if (NewV && NewV != V) {
11366 Worklist.push_back({{*NewV, CtxI}, S});
11367 continue;
11368 }
11369
11370 if (auto *CE = dyn_cast<ConstantExpr>(V)) {
11371 if (CE->getOpcode() == Instruction::ICmp)
11372 if (handleCmp(A, *CE, CE->getOperand(0), CE->getOperand(1),
11373 CmpInst::Predicate(CE->getPredicate()), II, Worklist))
11374 continue;
11375 }
11376
11377 if (auto *I = dyn_cast<Instruction>(V)) {
11378 if (simplifyInstruction(A, *I, II, Worklist, LivenessAAs))
11379 continue;
11380 }
11381
11382 if (V != InitialV || isa<Argument>(V))
11383 if (recurseForValue(A, IRPosition::value(*V), II.S))
11384 continue;
11385
11386 // If we haven't stripped anything we give up.
11387 if (V == InitialV && CtxI == getCtxI()) {
11388 indicatePessimisticFixpoint();
11389 return;
11390 }
11391
11392 addValue(A, getState(), *V, CtxI, S, getAnchorScope());
11393 } while (!Worklist.empty());
11394
11395 // If we actually used liveness information so we have to record a
11396 // dependence.
11397 for (auto &It : LivenessAAs)
11398 if (It.second.AnyDead)
11399 A.recordDependence(*It.second.LivenessAA, *this, DepClassTy::OPTIONAL);
11400 }
11401
11402 /// See AbstractAttribute::trackStatistics()
11403 void trackStatistics() const override {
11404 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11405 }
11406};
11407
11408struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11409 using Base = AAPotentialValuesImpl;
11410 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11411 : Base(IRP, A) {}
11412
11413 /// See AbstractAttribute::initialize(..).
11414 void initialize(Attributor &A) override {
11415 auto &Arg = cast<Argument>(getAssociatedValue());
11417 indicatePessimisticFixpoint();
11418 }
11419
11420 /// See AbstractAttribute::updateImpl(...).
11421 ChangeStatus updateImpl(Attributor &A) override {
11422 auto AssumedBefore = getAssumed();
11423
11424 unsigned ArgNo = getCalleeArgNo();
11425
11426 bool UsedAssumedInformation = false;
11428 auto CallSitePred = [&](AbstractCallSite ACS) {
11429 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11430 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11431 return false;
11432
11433 if (!A.getAssumedSimplifiedValues(CSArgIRP, this, Values,
11435 UsedAssumedInformation))
11436 return false;
11437
11438 return isValidState();
11439 };
11440
11441 if (!A.checkForAllCallSites(CallSitePred, *this,
11442 /* RequireAllCallSites */ true,
11443 UsedAssumedInformation))
11444 return indicatePessimisticFixpoint();
11445
11446 Function *Fn = getAssociatedFunction();
11447 bool AnyNonLocal = false;
11448 for (auto &It : Values) {
11449 if (isa<Constant>(It.getValue())) {
11450 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
11451 getAnchorScope());
11452 continue;
11453 }
11454 if (!AA::isDynamicallyUnique(A, *this, *It.getValue()))
11455 return indicatePessimisticFixpoint();
11456
11457 if (auto *Arg = dyn_cast<Argument>(It.getValue()))
11458 if (Arg->getParent() == Fn) {
11459 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
11460 getAnchorScope());
11461 continue;
11462 }
11463 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::Interprocedural,
11464 getAnchorScope());
11465 AnyNonLocal = true;
11466 }
11467 assert(!undefIsContained() && "Undef should be an explicit value!");
11468 if (AnyNonLocal)
11469 giveUpOnIntraprocedural(A);
11470
11471 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11472 : ChangeStatus::CHANGED;
11473 }
11474
11475 /// See AbstractAttribute::trackStatistics()
11476 void trackStatistics() const override {
11477 STATS_DECLTRACK_ARG_ATTR(potential_values)
11478 }
11479};
11480
11481struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11482 using Base = AAPotentialValuesFloating;
11483 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11484 : Base(IRP, A) {}
11485
11486 /// See AbstractAttribute::initialize(..).
11487 void initialize(Attributor &A) override {
11488 Function *F = getAssociatedFunction();
11489 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11490 indicatePessimisticFixpoint();
11491 return;
11492 }
11493
11494 for (Argument &Arg : F->args())
11495 if (Arg.hasReturnedAttr()) {
11496 addValue(A, getState(), Arg, nullptr, AA::AnyScope, F);
11497 ReturnedArg = &Arg;
11498 break;
11499 }
11500 if (!A.isFunctionIPOAmendable(*F) ||
11501 A.hasSimplificationCallback(getIRPosition())) {
11502 if (!ReturnedArg)
11503 indicatePessimisticFixpoint();
11504 else
11505 indicateOptimisticFixpoint();
11506 }
11507 }
11508
11509 /// See AbstractAttribute::updateImpl(...).
11510 ChangeStatus updateImpl(Attributor &A) override {
11511 auto AssumedBefore = getAssumed();
11512 bool UsedAssumedInformation = false;
11513
11515 Function *AnchorScope = getAnchorScope();
11516 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11517 bool AddValues) {
11519 Values.clear();
11520 if (!A.getAssumedSimplifiedValues(IRPosition::value(V), this, Values, S,
11521 UsedAssumedInformation,
11522 /* RecurseForSelectAndPHI */ true))
11523 return false;
11524 if (!AddValues)
11525 continue;
11526 for (const AA::ValueAndContext &VAC : Values)
11527 addValue(A, getState(), *VAC.getValue(),
11528 VAC.getCtxI() ? VAC.getCtxI() : CtxI, S, AnchorScope);
11529 }
11530 return true;
11531 };
11532
11533 if (ReturnedArg) {
11534 HandleReturnedValue(*ReturnedArg, nullptr, true);
11535 } else {
11536 auto RetInstPred = [&](Instruction &RetI) {
11537 bool AddValues = true;
11538 if (isa<PHINode>(RetI.getOperand(0)) ||
11539 isa<SelectInst>(RetI.getOperand(0))) {
11540 addValue(A, getState(), *RetI.getOperand(0), &RetI, AA::AnyScope,
11541 AnchorScope);
11542 AddValues = false;
11543 }
11544 return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);
11545 };
11546
11547 if (!A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
11548 UsedAssumedInformation,
11549 /* CheckBBLivenessOnly */ true))
11550 return indicatePessimisticFixpoint();
11551 }
11552
11553 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11554 : ChangeStatus::CHANGED;
11555 }
11556
11557 void addValue(Attributor &A, StateType &State, Value &V,
11558 const Instruction *CtxI, AA::ValueScope S,
11559 Function *AnchorScope) const override {
11560 Function *F = getAssociatedFunction();
11561 if (auto *CB = dyn_cast<CallBase>(&V))
11562 if (CB->getCalledOperand() == F)
11563 return;
11564 Base::addValue(A, State, V, CtxI, S, AnchorScope);
11565 }
11566
11567 ChangeStatus manifest(Attributor &A) override {
11568 if (ReturnedArg)
11569 return ChangeStatus::UNCHANGED;
11571 if (!getAssumedSimplifiedValues(A, Values, AA::ValueScope::Intraprocedural,
11572 /* RecurseForSelectAndPHI */ true))
11573 return ChangeStatus::UNCHANGED;
11574 Value *NewVal = getSingleValue(A, *this, getIRPosition(), Values);
11575 if (!NewVal)
11576 return ChangeStatus::UNCHANGED;
11577
11578 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11579 if (auto *Arg = dyn_cast<Argument>(NewVal)) {
11580 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11581 "Number of function with unique return");
11582 Changed |= A.manifestAttrs(
11584 {Attribute::get(Arg->getContext(), Attribute::Returned)});
11585 STATS_DECLTRACK_ARG_ATTR(returned);
11586 }
11587
11588 auto RetInstPred = [&](Instruction &RetI) {
11589 Value *RetOp = RetI.getOperand(0);
11590 if (isa<UndefValue>(RetOp) || RetOp == NewVal)
11591 return true;
11592 if (AA::isValidAtPosition({*NewVal, RetI}, A.getInfoCache()))
11593 if (A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))
11594 Changed = ChangeStatus::CHANGED;
11595 return true;
11596 };
11597 bool UsedAssumedInformation = false;
11598 (void)A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
11599 UsedAssumedInformation,
11600 /* CheckBBLivenessOnly */ true);
11601 return Changed;
11602 }
11603
11604 ChangeStatus indicatePessimisticFixpoint() override {
11606 }
11607
11608 /// See AbstractAttribute::trackStatistics()
11609 void trackStatistics() const override{
11610 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11611
11612 /// The argumented with an existing `returned` attribute.
11613 Argument *ReturnedArg = nullptr;
11614};
11615
11616struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11617 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11618 : AAPotentialValuesImpl(IRP, A) {}
11619
11620 /// See AbstractAttribute::updateImpl(...).
11621 ChangeStatus updateImpl(Attributor &A) override {
11622 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11623 "not be called");
11624 }
11625
11626 /// See AbstractAttribute::trackStatistics()
11627 void trackStatistics() const override {
11628 STATS_DECLTRACK_FN_ATTR(potential_values)
11629 }
11630};
11631
11632struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11633 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11634 : AAPotentialValuesFunction(IRP, A) {}
11635
11636 /// See AbstractAttribute::trackStatistics()
11637 void trackStatistics() const override {
11638 STATS_DECLTRACK_CS_ATTR(potential_values)
11639 }
11640};
11641
11642struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11643 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11644 : AAPotentialValuesImpl(IRP, A) {}
11645
11646 /// See AbstractAttribute::updateImpl(...).
11647 ChangeStatus updateImpl(Attributor &A) override {
11648 auto AssumedBefore = getAssumed();
11649
11650 Function *Callee = getAssociatedFunction();
11651 if (!Callee)
11652 return indicatePessimisticFixpoint();
11653
11654 bool UsedAssumedInformation = false;
11655 auto *CB = cast<CallBase>(getCtxI());
11656 if (CB->isMustTailCall() &&
11657 !A.isAssumedDead(IRPosition::inst(*CB), this, nullptr,
11658 UsedAssumedInformation))
11659 return indicatePessimisticFixpoint();
11660
11662 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
11663 Values, AA::Intraprocedural,
11664 UsedAssumedInformation))
11665 return indicatePessimisticFixpoint();
11666
11667 Function *Caller = CB->getCaller();
11668
11669 bool AnyNonLocal = false;
11670 for (auto &It : Values) {
11671 Value *V = It.getValue();
11672 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11673 V, *CB, *this, UsedAssumedInformation);
11674 if (!CallerV.has_value()) {
11675 // Nothing to do as long as no value was determined.
11676 continue;
11677 }
11678 V = *CallerV ? *CallerV : V;
11679 if (AA::isDynamicallyUnique(A, *this, *V) &&
11680 AA::isValidInScope(*V, Caller)) {
11681 if (*CallerV) {
11683 IRPosition IRP = IRPosition::value(*V);
11684 if (auto *Arg = dyn_cast<Argument>(V))
11685 if (Arg->getParent() == CB->getCalledOperand())
11686 IRP = IRPosition::callsite_argument(*CB, Arg->getArgNo());
11687 if (recurseForValue(A, IRP, AA::AnyScope))
11688 continue;
11689 }
11690 addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
11691 } else {
11692 AnyNonLocal = true;
11693 break;
11694 }
11695 }
11696 if (AnyNonLocal) {
11697 Values.clear();
11698 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
11699 Values, AA::Interprocedural,
11700 UsedAssumedInformation))
11701 return indicatePessimisticFixpoint();
11702 AnyNonLocal = false;
11704 for (auto &It : Values) {
11705 Value *V = It.getValue();
11706 if (!AA::isDynamicallyUnique(A, *this, *V))
11707 return indicatePessimisticFixpoint();
11708 if (AA::isValidInScope(*V, Caller)) {
11709 addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
11710 } else {
11711 AnyNonLocal = true;
11712 addValue(A, getState(), *V, CB, AA::Interprocedural,
11713 getAnchorScope());
11714 }
11715 }
11716 if (AnyNonLocal)
11717 giveUpOnIntraprocedural(A);
11718 }
11719 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11720 : ChangeStatus::CHANGED;
11721 }
11722
11723 ChangeStatus indicatePessimisticFixpoint() override {
11724 return AAPotentialValues::indicatePessimisticFixpoint();
11725 }
11726
11727 /// See AbstractAttribute::trackStatistics()
11728 void trackStatistics() const override {
11729 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11730 }
11731};
11732
11733struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11734 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11735 : AAPotentialValuesFloating(IRP, A) {}
11736
11737 /// See AbstractAttribute::trackStatistics()
11738 void trackStatistics() const override {
11739 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11740 }
11741};
11742} // namespace
11743
11744/// ---------------------- Assumption Propagation ------------------------------
11745namespace {
11746struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11747 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11748 const DenseSet<StringRef> &Known)
11749 : AAAssumptionInfo(IRP, A, Known) {}
11750
11751 /// See AbstractAttribute::manifest(...).
11752 ChangeStatus manifest(Attributor &A) override {
11753 // Don't manifest a universal set if it somehow made it here.
11754 if (getKnown().isUniversal())
11755 return ChangeStatus::UNCHANGED;
11756
11757 const IRPosition &IRP = getIRPosition();
11758 return A.manifestAttrs(
11759 IRP,
11761 llvm::join(getAssumed().getSet(), ",")),
11762 /* ForceReplace */ true);
11763 }
11764
11765 bool hasAssumption(const StringRef Assumption) const override {
11766 return isValidState() && setContains(Assumption);
11767 }
11768
11769 /// See AbstractAttribute::getAsStr()
11770 const std::string getAsStr(Attributor *A) const override {
11771 const SetContents &Known = getKnown();
11772 const SetContents &Assumed = getAssumed();
11773
11774 const std::string KnownStr =
11775 llvm::join(Known.getSet().begin(), Known.getSet().end(), ",");
11776 const std::string AssumedStr =
11777 (Assumed.isUniversal())
11778 ? "Universal"
11779 : llvm::join(Assumed.getSet().begin(), Assumed.getSet().end(), ",");
11780
11781 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11782 }
11783};
11784
11785/// Propagates assumption information from parent functions to all of their
11786/// successors. An assumption can be propagated if the containing function
11787/// dominates the called function.
11788///
11789/// We start with a "known" set of assumptions already valid for the associated
11790/// function and an "assumed" set that initially contains all possible
11791/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11792/// contents as concrete values are known. The concrete values are seeded by the
11793/// first nodes that are either entries into the call graph, or contains no
11794/// assumptions. Each node is updated as the intersection of the assumed state
11795/// with all of its predecessors.
11796struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11797 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11798 : AAAssumptionInfoImpl(IRP, A,
11799 getAssumptions(*IRP.getAssociatedFunction())) {}
11800
11801 /// See AbstractAttribute::updateImpl(...).
11802 ChangeStatus updateImpl(Attributor &A) override {
11803 bool Changed = false;
11804
11805 auto CallSitePred = [&](AbstractCallSite ACS) {
11806 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11807 *this, IRPosition::callsite_function(*ACS.getInstruction()),
11808 DepClassTy::REQUIRED);
11809 if (!AssumptionAA)
11810 return false;
11811 // Get the set of assumptions shared by all of this function's callers.
11812 Changed |= getIntersection(AssumptionAA->getAssumed());
11813 return !getAssumed().empty() || !getKnown().empty();
11814 };
11815
11816 bool UsedAssumedInformation = false;
11817 // Get the intersection of all assumptions held by this node's predecessors.
11818 // If we don't know all the call sites then this is either an entry into the
11819 // call graph or an empty node. This node is known to only contain its own
11820 // assumptions and can be propagated to its successors.
11821 if (!A.checkForAllCallSites(CallSitePred, *this, true,
11822 UsedAssumedInformation))
11823 return indicatePessimisticFixpoint();
11824
11825 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11826 }
11827
11828 void trackStatistics() const override {}
11829};
11830
11831/// Assumption Info defined for call sites.
11832struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11833
11834 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11835 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11836
11837 /// See AbstractAttribute::initialize(...).
11838 void initialize(Attributor &A) override {
11839 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
11840 A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
11841 }
11842
11843 /// See AbstractAttribute::updateImpl(...).
11844 ChangeStatus updateImpl(Attributor &A) override {
11845 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
11846 auto *AssumptionAA =
11847 A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
11848 if (!AssumptionAA)
11849 return indicatePessimisticFixpoint();
11850 bool Changed = getIntersection(AssumptionAA->getAssumed());
11851 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11852 }
11853
11854 /// See AbstractAttribute::trackStatistics()
11855 void trackStatistics() const override {}
11856
11857private:
11858 /// Helper to initialized the known set as all the assumptions this call and
11859 /// the callee contain.
11860 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
11861 const CallBase &CB = cast<CallBase>(IRP.getAssociatedValue());
11862 auto Assumptions = getAssumptions(CB);
11863 if (const Function *F = CB.getCaller())
11864 set_union(Assumptions, getAssumptions(*F));
11865 if (Function *F = IRP.getAssociatedFunction())
11866 set_union(Assumptions, getAssumptions(*F));
11867 return Assumptions;
11868 }
11869};
11870} // namespace
11871
11873 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
11874 A.getOrCreateAAFor<AACallEdges>(IRPosition::function(**I))));
11875}
11876
11878
11879/// ------------------------ UnderlyingObjects ---------------------------------
11880
11881namespace {
11882struct AAUnderlyingObjectsImpl
11883 : StateWrapper<BooleanState, AAUnderlyingObjects> {
11885 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
11886
11887 /// See AbstractAttribute::getAsStr().
11888 const std::string getAsStr(Attributor *A) const override {
11889 return std::string("UnderlyingObjects ") +
11890 (isValidState()
11891 ? (std::string("inter #") +
11892 std::to_string(InterAssumedUnderlyingObjects.size()) +
11893 " objs" + std::string(", intra #") +
11894 std::to_string(IntraAssumedUnderlyingObjects.size()) +
11895 " objs")
11896 : "<invalid>");
11897 }
11898
11899 /// See AbstractAttribute::trackStatistics()
11900 void trackStatistics() const override {}
11901
11902 /// See AbstractAttribute::updateImpl(...).
11903 ChangeStatus updateImpl(Attributor &A) override {
11904 auto &Ptr = getAssociatedValue();
11905
11906 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
11908 bool UsedAssumedInformation = false;
11909 SmallPtrSet<Value *, 8> SeenObjects;
11911
11912 if (!A.getAssumedSimplifiedValues(IRPosition::value(Ptr), *this, Values,
11913 Scope, UsedAssumedInformation))
11914 return UnderlyingObjects.insert(&Ptr);
11915
11916 bool Changed = false;
11917
11918 for (unsigned I = 0; I < Values.size(); ++I) {
11919 auto &VAC = Values[I];
11920 auto *Obj = VAC.getValue();
11921 Value *UO = getUnderlyingObject(Obj);
11922 if (UO && UO != VAC.getValue() && SeenObjects.insert(UO).second) {
11923 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
11924 *this, IRPosition::value(*UO), DepClassTy::OPTIONAL);
11925 auto Pred = [&Values](Value &V) {
11926 Values.emplace_back(V, nullptr);
11927 return true;
11928 };
11929
11930 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
11932 "The forall call should not return false at this position");
11933
11934 continue;
11935 }
11936
11937 if (isa<SelectInst>(Obj)) {
11938 Changed |= handleIndirect(A, *Obj, UnderlyingObjects, Scope);
11939 continue;
11940 }
11941 if (auto *PHI = dyn_cast<PHINode>(Obj)) {
11942 // Explicitly look through PHIs as we do not care about dynamically
11943 // uniqueness.
11944 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
11945 Changed |= handleIndirect(A, *PHI->getIncomingValue(u),
11946 UnderlyingObjects, Scope);
11947 }
11948 continue;
11949 }
11950
11951 Changed |= UnderlyingObjects.insert(Obj);
11952 }
11953
11954 return Changed;
11955 };
11956
11957 bool Changed = false;
11958 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
11959 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
11960
11961 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11962 }
11963
11964 bool forallUnderlyingObjects(
11965 function_ref<bool(Value &)> Pred,
11966 AA::ValueScope Scope = AA::Interprocedural) const override {
11967 if (!isValidState())
11968 return Pred(getAssociatedValue());
11969
11970 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
11971 ? IntraAssumedUnderlyingObjects
11972 : InterAssumedUnderlyingObjects;
11973 for (Value *Obj : AssumedUnderlyingObjects)
11974 if (!Pred(*Obj))
11975 return false;
11976
11977 return true;
11978 }
11979
11980private:
11981 /// Handle the case where the value is not the actual underlying value, such
11982 /// as a phi node or a select instruction.
11983 bool handleIndirect(Attributor &A, Value &V,
11984 SmallSetVector<Value *, 8> &UnderlyingObjects,
11985 AA::ValueScope Scope) {
11986 bool Changed = false;
11987 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
11988 *this, IRPosition::value(V), DepClassTy::OPTIONAL);
11989 auto Pred = [&](Value &V) {
11990 Changed |= UnderlyingObjects.insert(&V);
11991 return true;
11992 };
11993 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
11995 "The forall call should not return false at this position");
11996 return Changed;
11997 }
11998
11999 /// All the underlying objects collected so far via intra procedural scope.
12000 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12001 /// All the underlying objects collected so far via inter procedural scope.
12002 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12003};
12004
12005struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12006 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12007 : AAUnderlyingObjectsImpl(IRP, A) {}
12008};
12009
12010struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12011 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12012 : AAUnderlyingObjectsImpl(IRP, A) {}
12013};
12014
12015struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12016 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12017 : AAUnderlyingObjectsImpl(IRP, A) {}
12018};
12019
12020struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12021 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12022 : AAUnderlyingObjectsImpl(IRP, A) {}
12023};
12024
12025struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12026 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12027 : AAUnderlyingObjectsImpl(IRP, A) {}
12028};
12029
12030struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12031 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12032 : AAUnderlyingObjectsImpl(IRP, A) {}
12033};
12034
12035struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12036 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12037 : AAUnderlyingObjectsImpl(IRP, A) {}
12038};
12039} // namespace
12040
12041/// ------------------------ Global Value Info -------------------------------
12042namespace {
12043struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12044 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12045 : AAGlobalValueInfo(IRP, A) {}
12046
12047 /// See AbstractAttribute::initialize(...).
12048 void initialize(Attributor &A) override {}
12049
12050 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12052 Instruction *UInst = dyn_cast<Instruction>(U.getUser());
12053 if (!UInst) {
12054 Follow = true;
12055 return true;
12056 }
12057
12058 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12059 << *UInst << "\n");
12060
12061 if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
12062 int Idx = &Cmp->getOperandUse(0) == &U;
12063 if (isa<Constant>(Cmp->getOperand(Idx)))
12064 return true;
12065 return U == &getAnchorValue();
12066 }
12067
12068 // Explicitly catch return instructions.
12069 if (isa<ReturnInst>(UInst)) {
12070 auto CallSitePred = [&](AbstractCallSite ACS) {
12071 Worklist.push_back(ACS.getInstruction());
12072 return true;
12073 };
12074 bool UsedAssumedInformation = false;
12075 // TODO: We should traverse the uses or add a "non-call-site" CB.
12076 if (!A.checkForAllCallSites(CallSitePred, *UInst->getFunction(),
12077 /*RequireAllCallSites=*/true, this,
12078 UsedAssumedInformation))
12079 return false;
12080 return true;
12081 }
12082
12083 // For now we only use special logic for call sites. However, the tracker
12084 // itself knows about a lot of other non-capturing cases already.
12085 auto *CB = dyn_cast<CallBase>(UInst);
12086 if (!CB)
12087 return false;
12088 // Direct calls are OK uses.
12089 if (CB->isCallee(&U))
12090 return true;
12091 // Non-argument uses are scary.
12092 if (!CB->isArgOperand(&U))
12093 return false;
12094 // TODO: Iterate callees.
12095 auto *Fn = dyn_cast<Function>(CB->getCalledOperand());
12096 if (!Fn || !A.isFunctionIPOAmendable(*Fn))
12097 return false;
12098
12099 unsigned ArgNo = CB->getArgOperandNo(&U);
12100 Worklist.push_back(Fn->getArg(ArgNo));
12101 return true;
12102 }
12103
12104 ChangeStatus updateImpl(Attributor &A) override {
12105 unsigned NumUsesBefore = Uses.size();
12106
12109 Worklist.push_back(&getAnchorValue());
12110
12111 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12112 Uses.insert(&U);
12113 switch (DetermineUseCaptureKind(U, nullptr)) {
12114 case UseCaptureKind::NO_CAPTURE:
12115 return checkUse(A, U, Follow, Worklist);
12116 case UseCaptureKind::MAY_CAPTURE:
12117 return checkUse(A, U, Follow, Worklist);
12118 case UseCaptureKind::PASSTHROUGH:
12119 Follow = true;
12120 return true;
12121 }
12122 return true;
12123 };
12124 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12125 Uses.insert(&OldU);
12126 return true;
12127 };
12128
12129 while (!Worklist.empty()) {
12130 const Value *V = Worklist.pop_back_val();
12131 if (!Visited.insert(V).second)
12132 continue;
12133 if (!A.checkForAllUses(UsePred, *this, *V,
12134 /* CheckBBLivenessOnly */ true,
12135 DepClassTy::OPTIONAL,
12136 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12137 return indicatePessimisticFixpoint();
12138 }
12139 }
12140
12141 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12142 : ChangeStatus::CHANGED;
12143 }
12144
12145 bool isPotentialUse(const Use &U) const override {
12146 return !isValidState() || Uses.contains(&U);
12147 }
12148
12149 /// See AbstractAttribute::manifest(...).
12150 ChangeStatus manifest(Attributor &A) override {
12151 return ChangeStatus::UNCHANGED;
12152 }
12153
12154 /// See AbstractAttribute::getAsStr().
12155 const std::string getAsStr(Attributor *A) const override {
12156 return "[" + std::to_string(Uses.size()) + " uses]";
12157 }
12158
12159 void trackStatistics() const override {
12160 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12161 }
12162
12163private:
12164 /// Set of (transitive) uses of this GlobalValue.
12166};
12167} // namespace
12168
12169/// ------------------------ Indirect Call Info -------------------------------
12170namespace {
12171struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12172 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12173 : AAIndirectCallInfo(IRP, A) {}
12174
12175 /// See AbstractAttribute::initialize(...).
12176 void initialize(Attributor &A) override {
12177 auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);
12178 if (!MD && !A.isClosedWorldModule())
12179 return;
12180
12181 if (MD) {
12182 for (const auto &Op : MD->operands())
12183 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(Op))
12184 PotentialCallees.insert(Callee);
12185 } else if (A.isClosedWorldModule()) {
12186 ArrayRef<Function *> IndirectlyCallableFunctions =
12187 A.getInfoCache().getIndirectlyCallableFunctions(A);
12188 PotentialCallees.insert(IndirectlyCallableFunctions.begin(),
12189 IndirectlyCallableFunctions.end());
12190 }
12191
12192 if (PotentialCallees.empty())
12193 indicateOptimisticFixpoint();
12194 }
12195
12196 ChangeStatus updateImpl(Attributor &A) override {
12197 CallBase *CB = cast<CallBase>(getCtxI());
12198 const Use &CalleeUse = CB->getCalledOperandUse();
12199 Value *FP = CB->getCalledOperand();
12200
12201 SmallSetVector<Function *, 4> AssumedCalleesNow;
12202 bool AllCalleesKnownNow = AllCalleesKnown;
12203
12204 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12205 bool &UsedAssumedInformation) {
12206 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12207 *this, IRPosition::value(PotentialCallee), DepClassTy::OPTIONAL);
12208 if (!GIAA || GIAA->isPotentialUse(CalleeUse))
12209 return true;
12210 UsedAssumedInformation = !GIAA->isAtFixpoint();
12211 return false;
12212 };
12213
12214 auto AddPotentialCallees = [&]() {
12215 for (auto *PotentialCallee : PotentialCallees) {
12216 bool UsedAssumedInformation = false;
12217 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12218 AssumedCalleesNow.insert(PotentialCallee);
12219 }
12220 };
12221
12222 // Use simplification to find potential callees, if !callees was present,
12223 // fallback to that set if necessary.
12224 bool UsedAssumedInformation = false;
12226 if (!A.getAssumedSimplifiedValues(IRPosition::value(*FP), this, Values,
12227 AA::ValueScope::AnyScope,
12228 UsedAssumedInformation)) {
12229 if (PotentialCallees.empty())
12230 return indicatePessimisticFixpoint();
12231 AddPotentialCallees();
12232 }
12233
12234 // Try to find a reason for \p Fn not to be a potential callee. If none was
12235 // found, add it to the assumed callees set.
12236 auto CheckPotentialCallee = [&](Function &Fn) {
12237 if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))
12238 return false;
12239
12240 auto &CachedResult = FilterResults[&Fn];
12241 if (CachedResult.has_value())
12242 return CachedResult.value();
12243
12244 bool UsedAssumedInformation = false;
12245 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12246 if (!UsedAssumedInformation)
12247 CachedResult = false;
12248 return false;
12249 }
12250
12251 int NumFnArgs = Fn.arg_size();
12252 int NumCBArgs = CB->arg_size();
12253
12254 // Check if any excess argument (which we fill up with poison) is known to
12255 // be UB on undef.
12256 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12257 bool IsKnown = false;
12258 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12259 A, this, IRPosition::argument(*Fn.getArg(I)),
12260 DepClassTy::OPTIONAL, IsKnown)) {
12261 if (IsKnown)
12262 CachedResult = false;
12263 return false;
12264 }
12265 }
12266
12267 CachedResult = true;
12268 return true;
12269 };
12270
12271 // Check simplification result, prune known UB callees, also restrict it to
12272 // the !callees set, if present.
12273 for (auto &VAC : Values) {
12274 if (isa<UndefValue>(VAC.getValue()))
12275 continue;
12276 if (isa<ConstantPointerNull>(VAC.getValue()) &&
12277 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12278 continue;
12279 // TODO: Check for known UB, e.g., poison + noundef.
12280 if (auto *VACFn = dyn_cast<Function>(VAC.getValue())) {
12281 if (CheckPotentialCallee(*VACFn))
12282 AssumedCalleesNow.insert(VACFn);
12283 continue;
12284 }
12285 if (!PotentialCallees.empty()) {
12286 AddPotentialCallees();
12287 break;
12288 }
12289 AllCalleesKnownNow = false;
12290 }
12291
12292 if (AssumedCalleesNow == AssumedCallees &&
12293 AllCalleesKnown == AllCalleesKnownNow)
12294 return ChangeStatus::UNCHANGED;
12295
12296 std::swap(AssumedCallees, AssumedCalleesNow);
12297 AllCalleesKnown = AllCalleesKnownNow;
12298 return ChangeStatus::CHANGED;
12299 }
12300
12301 /// See AbstractAttribute::manifest(...).
12302 ChangeStatus manifest(Attributor &A) override {
12303 // If we can't specialize at all, give up now.
12304 if (!AllCalleesKnown && AssumedCallees.empty())
12305 return ChangeStatus::UNCHANGED;
12306
12307 CallBase *CB = cast<CallBase>(getCtxI());
12308 bool UsedAssumedInformation = false;
12309 if (A.isAssumedDead(*CB, this, /*LivenessAA=*/nullptr,
12310 UsedAssumedInformation))
12311 return ChangeStatus::UNCHANGED;
12312
12313 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12314 Value *FP = CB->getCalledOperand();
12315 if (FP->getType()->getPointerAddressSpace())
12316 FP = new AddrSpaceCastInst(FP, PointerType::get(FP->getType(), 0),
12317 FP->getName() + ".as0", CB->getIterator());
12318
12319 bool CBIsVoid = CB->getType()->isVoidTy();
12321 FunctionType *CSFT = CB->getFunctionType();
12322 SmallVector<Value *> CSArgs(CB->arg_begin(), CB->arg_end());
12323
12324 // If we know all callees and there are none, the call site is (effectively)
12325 // dead (or UB).
12326 if (AssumedCallees.empty()) {
12327 assert(AllCalleesKnown &&
12328 "Expected all callees to be known if there are none.");
12329 A.changeToUnreachableAfterManifest(CB);
12330 return ChangeStatus::CHANGED;
12331 }
12332
12333 // Special handling for the single callee case.
12334 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12335 auto *NewCallee = AssumedCallees.front();
12336 if (isLegalToPromote(*CB, NewCallee)) {
12337 promoteCall(*CB, NewCallee, nullptr);
12338 return ChangeStatus::CHANGED;
12339 }
12340 Instruction *NewCall =
12341 CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
12342 CB->getName(), CB->getIterator());
12343 if (!CBIsVoid)
12344 A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall);
12345 A.deleteAfterManifest(*CB);
12346 return ChangeStatus::CHANGED;
12347 }
12348
12349 // For each potential value we create a conditional
12350 //
12351 // ```
12352 // if (ptr == value) value(args);
12353 // else ...
12354 // ```
12355 //
12356 bool SpecializedForAnyCallees = false;
12357 bool SpecializedForAllCallees = AllCalleesKnown;
12358 ICmpInst *LastCmp = nullptr;
12359 SmallVector<Function *, 8> SkippedAssumedCallees;
12361 for (Function *NewCallee : AssumedCallees) {
12362 if (!A.shouldSpecializeCallSiteForCallee(*this, *CB, *NewCallee)) {
12363 SkippedAssumedCallees.push_back(NewCallee);
12364 SpecializedForAllCallees = false;
12365 continue;
12366 }
12367 SpecializedForAnyCallees = true;
12368
12369 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12370 Instruction *ThenTI =
12371 SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false);
12372 BasicBlock *CBBB = CB->getParent();
12373 A.registerManifestAddedBasicBlock(*ThenTI->getParent());
12374 A.registerManifestAddedBasicBlock(*IP->getParent());
12375 auto *SplitTI = cast<BranchInst>(LastCmp->getNextNode());
12376 BasicBlock *ElseBB;
12377 if (&*IP == CB) {
12378 ElseBB = BasicBlock::Create(ThenTI->getContext(), "",
12379 ThenTI->getFunction(), CBBB);
12380 A.registerManifestAddedBasicBlock(*ElseBB);
12381 IP = BranchInst::Create(CBBB, ElseBB)->getIterator();
12382 SplitTI->replaceUsesOfWith(CBBB, ElseBB);
12383 } else {
12384 ElseBB = IP->getParent();
12385 ThenTI->replaceUsesOfWith(ElseBB, CBBB);
12386 }
12387 CastInst *RetBC = nullptr;
12388 CallInst *NewCall = nullptr;
12389 if (isLegalToPromote(*CB, NewCallee)) {
12390 auto *CBClone = cast<CallBase>(CB->clone());
12391 CBClone->insertBefore(ThenTI);
12392 NewCall = &cast<CallInst>(promoteCall(*CBClone, NewCallee, &RetBC));
12393 } else {
12394 NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
12395 CB->getName(), ThenTI->getIterator());
12396 }
12397 NewCalls.push_back({NewCall, RetBC});
12398 }
12399
12400 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12401 if (!AllCalleesKnown)
12402 return ChangeStatus::UNCHANGED;
12403 MDBuilder MDB(IndirectCB.getContext());
12404 MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);
12405 IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);
12406 return ChangeStatus::CHANGED;
12407 };
12408
12409 if (!SpecializedForAnyCallees)
12410 return AttachCalleeMetadata(*CB);
12411
12412 // Check if we need the fallback indirect call still.
12413 if (SpecializedForAllCallees) {
12415 LastCmp->eraseFromParent();
12416 new UnreachableInst(IP->getContext(), IP);
12417 IP->eraseFromParent();
12418 } else {
12419 auto *CBClone = cast<CallInst>(CB->clone());
12420 CBClone->setName(CB->getName());
12421 CBClone->insertBefore(*IP->getParent(), IP);
12422 NewCalls.push_back({CBClone, nullptr});
12423 AttachCalleeMetadata(*CBClone);
12424 }
12425
12426 // Check if we need a PHI to merge the results.
12427 if (!CBIsVoid) {
12428 auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(),
12429 CB->getName() + ".phi",
12431 for (auto &It : NewCalls) {
12432 CallBase *NewCall = It.first;
12433 Instruction *CallRet = It.second ? It.second : It.first;
12434 if (CallRet->getType() == CB->getType())
12435 PHI->addIncoming(CallRet, CallRet->getParent());
12436 else if (NewCall->getType()->isVoidTy())
12437 PHI->addIncoming(PoisonValue::get(CB->getType()),
12438 NewCall->getParent());
12439 else
12440 llvm_unreachable("Call return should match or be void!");
12441 }
12442 A.changeAfterManifest(IRPosition::callsite_returned(*CB), *PHI);
12443 }
12444
12445 A.deleteAfterManifest(*CB);
12446 Changed = ChangeStatus::CHANGED;
12447
12448 return Changed;
12449 }
12450
12451 /// See AbstractAttribute::getAsStr().
12452 const std::string getAsStr(Attributor *A) const override {
12453 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12454 " indirect call site with " + std::to_string(AssumedCallees.size()) +
12455 " functions";
12456 }
12457
12458 void trackStatistics() const override {
12459 if (AllCalleesKnown) {
12461 Eliminated, CallSites,
12462 "Number of indirect call sites eliminated via specialization")
12463 } else {
12464 STATS_DECLTRACK(Specialized, CallSites,
12465 "Number of indirect call sites specialized")
12466 }
12467 }
12468
12469 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12470 return isValidState() && AllCalleesKnown && all_of(AssumedCallees, CB);
12471 }
12472
12473private:
12474 /// Map to remember filter results.
12476
12477 /// If the !callee metadata was present, this set will contain all potential
12478 /// callees (superset).
12479 SmallSetVector<Function *, 4> PotentialCallees;
12480
12481 /// This set contains all currently assumed calllees, which might grow over
12482 /// time.
12483 SmallSetVector<Function *, 4> AssumedCallees;
12484
12485 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12486 /// if there could be others.
12487 bool AllCalleesKnown = true;
12488};
12489} // namespace
12490
12491/// ------------------------ Address Space ------------------------------------
12492namespace {
12493struct AAAddressSpaceImpl : public AAAddressSpace {
12494 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
12495 : AAAddressSpace(IRP, A) {}
12496
12497 int32_t getAddressSpace() const override {
12498 assert(isValidState() && "the AA is invalid");
12499 return AssumedAddressSpace;
12500 }
12501
12502 /// See AbstractAttribute::initialize(...).
12503 void initialize(Attributor &A) override {
12504 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
12505 "Associated value is not a pointer");
12506 }
12507
12508 ChangeStatus updateImpl(Attributor &A) override {
12509 int32_t OldAddressSpace = AssumedAddressSpace;
12510 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,
12511 DepClassTy::REQUIRED);
12512 auto Pred = [&](Value &Obj) {
12513 if (isa<UndefValue>(&Obj))
12514 return true;
12515 return takeAddressSpace(Obj.getType()->getPointerAddressSpace());
12516 };
12517
12518 if (!AUO->forallUnderlyingObjects(Pred))
12519 return indicatePessimisticFixpoint();
12520
12521 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
12522 : ChangeStatus::CHANGED;
12523 }
12524
12525 /// See AbstractAttribute::manifest(...).
12526 ChangeStatus manifest(Attributor &A) override {
12527 Value *AssociatedValue = &getAssociatedValue();
12528 Value *OriginalValue = peelAddrspacecast(AssociatedValue);
12529 if (getAddressSpace() == NoAddressSpace ||
12530 static_cast<uint32_t>(getAddressSpace()) ==
12531 getAssociatedType()->getPointerAddressSpace())
12532 return ChangeStatus::UNCHANGED;
12533
12534 Type *NewPtrTy = PointerType::get(getAssociatedType()->getContext(),
12535 static_cast<uint32_t>(getAddressSpace()));
12536 bool UseOriginalValue =
12537 OriginalValue->getType()->getPointerAddressSpace() ==
12538 static_cast<uint32_t>(getAddressSpace());
12539
12540 bool Changed = false;
12541
12542 auto MakeChange = [&](Instruction *I, Use &U) {
12543 Changed = true;
12544 if (UseOriginalValue) {
12545 A.changeUseAfterManifest(U, *OriginalValue);
12546 return;
12547 }
12548 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
12549 CastInst->insertBefore(cast<Instruction>(I));
12550 A.changeUseAfterManifest(U, *CastInst);
12551 };
12552
12553 auto Pred = [&](const Use &U, bool &) {
12554 if (U.get() != AssociatedValue)
12555 return true;
12556 auto *Inst = dyn_cast<Instruction>(U.getUser());
12557 if (!Inst)
12558 return true;
12559 // This is a WA to make sure we only change uses from the corresponding
12560 // CGSCC if the AA is run on CGSCC instead of the entire module.
12561 if (!A.isRunOn(Inst->getFunction()))
12562 return true;
12563 if (isa<LoadInst>(Inst))
12564 MakeChange(Inst, const_cast<Use &>(U));
12565 if (isa<StoreInst>(Inst)) {
12566 // We only make changes if the use is the pointer operand.
12567 if (U.getOperandNo() == 1)
12568 MakeChange(Inst, const_cast<Use &>(U));
12569 }
12570 return true;
12571 };
12572
12573 // It doesn't matter if we can't check all uses as we can simply
12574 // conservatively ignore those that can not be visited.
12575 (void)A.checkForAllUses(Pred, *this, getAssociatedValue(),
12576 /* CheckBBLivenessOnly */ true);
12577
12578 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12579 }
12580
12581 /// See AbstractAttribute::getAsStr().
12582 const std::string getAsStr(Attributor *A) const override {
12583 if (!isValidState())
12584 return "addrspace(<invalid>)";
12585 return "addrspace(" +
12586 (AssumedAddressSpace == NoAddressSpace
12587 ? "none"
12588 : std::to_string(AssumedAddressSpace)) +
12589 ")";
12590 }
12591
12592private:
12593 int32_t AssumedAddressSpace = NoAddressSpace;
12594
12595 bool takeAddressSpace(int32_t AS) {
12596 if (AssumedAddressSpace == NoAddressSpace) {
12597 AssumedAddressSpace = AS;
12598 return true;
12599 }
12600 return AssumedAddressSpace == AS;
12601 }
12602
12603 static Value *peelAddrspacecast(Value *V) {
12604 if (auto *I = dyn_cast<AddrSpaceCastInst>(V))
12605 return peelAddrspacecast(I->getPointerOperand());
12606 if (auto *C = dyn_cast<ConstantExpr>(V))
12607 if (C->getOpcode() == Instruction::AddrSpaceCast)
12608 return peelAddrspacecast(C->getOperand(0));
12609 return V;
12610 }
12611};
12612
12613struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
12614 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
12615 : AAAddressSpaceImpl(IRP, A) {}
12616
12617 void trackStatistics() const override {
12619 }
12620};
12621
12622struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
12623 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
12624 : AAAddressSpaceImpl(IRP, A) {}
12625
12626 /// See AbstractAttribute::initialize(...).
12627 void initialize(Attributor &A) override {
12628 // TODO: we don't rewrite function argument for now because it will need to
12629 // rewrite the function signature and all call sites.
12630 (void)indicatePessimisticFixpoint();
12631 }
12632
12633 void trackStatistics() const override {
12634 STATS_DECLTRACK_FNRET_ATTR(addrspace);
12635 }
12636};
12637
12638struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
12639 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
12640 : AAAddressSpaceImpl(IRP, A) {}
12641
12642 void trackStatistics() const override {
12643 STATS_DECLTRACK_CSRET_ATTR(addrspace);
12644 }
12645};
12646
12647struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
12648 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
12649 : AAAddressSpaceImpl(IRP, A) {}
12650
12651 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
12652};
12653
12654struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
12655 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
12656 : AAAddressSpaceImpl(IRP, A) {}
12657
12658 /// See AbstractAttribute::initialize(...).
12659 void initialize(Attributor &A) override {
12660 // TODO: we don't rewrite call site argument for now because it will need to
12661 // rewrite the function signature of the callee.
12662 (void)indicatePessimisticFixpoint();
12663 }
12664
12665 void trackStatistics() const override {
12666 STATS_DECLTRACK_CSARG_ATTR(addrspace);
12667 }
12668};
12669} // namespace
12670
12671/// ----------- Allocation Info ----------
12672namespace {
12673struct AAAllocationInfoImpl : public AAAllocationInfo {
12674 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
12675 : AAAllocationInfo(IRP, A) {}
12676
12677 std::optional<TypeSize> getAllocatedSize() const override {
12678 assert(isValidState() && "the AA is invalid");
12679 return AssumedAllocatedSize;
12680 }
12681
12682 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
12683 const DataLayout &DL) {
12684
12685 // TODO: implement case for malloc like instructions
12686 switch (I->getOpcode()) {
12687 case Instruction::Alloca: {
12688 AllocaInst *AI = cast<AllocaInst>(I);
12689 return AI->getAllocationSize(DL);
12690 }
12691 default:
12692 return std::nullopt;
12693 }
12694 }
12695
12696 ChangeStatus updateImpl(Attributor &A) override {
12697
12698 const IRPosition &IRP = getIRPosition();
12699 Instruction *I = IRP.getCtxI();
12700
12701 // TODO: update check for malloc like calls
12702 if (!isa<AllocaInst>(I))
12703 return indicatePessimisticFixpoint();
12704
12705 bool IsKnownNoCapture;
12706 if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
12707 A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
12708 return indicatePessimisticFixpoint();
12709
12710 const AAPointerInfo *PI =
12711 A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
12712
12713 if (!PI)
12714 return indicatePessimisticFixpoint();
12715
12716 if (!PI->getState().isValidState())
12717 return indicatePessimisticFixpoint();
12718
12719 const DataLayout &DL = A.getDataLayout();
12720 const auto AllocationSize = findInitialAllocationSize(I, DL);
12721
12722 // If allocation size is nullopt, we give up.
12723 if (!AllocationSize)
12724 return indicatePessimisticFixpoint();
12725
12726 // For zero sized allocations, we give up.
12727 // Since we can't reduce further
12728 if (*AllocationSize == 0)
12729 return indicatePessimisticFixpoint();
12730
12731 int64_t BinSize = PI->numOffsetBins();
12732
12733 // TODO: implement for multiple bins
12734 if (BinSize > 1)
12735 return indicatePessimisticFixpoint();
12736
12737 if (BinSize == 0) {
12738 auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false));
12739 if (!changeAllocationSize(NewAllocationSize))
12740 return ChangeStatus::UNCHANGED;
12741 return ChangeStatus::CHANGED;
12742 }
12743
12744 // TODO: refactor this to be part of multiple bin case
12745 const auto &It = PI->begin();
12746
12747 // TODO: handle if Offset is not zero
12748 if (It->first.Offset != 0)
12749 return indicatePessimisticFixpoint();
12750
12751 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
12752
12753 if (SizeOfBin >= *AllocationSize)
12754 return indicatePessimisticFixpoint();
12755
12756 auto NewAllocationSize =
12757 std::optional<TypeSize>(TypeSize(SizeOfBin * 8, false));
12758
12759 if (!changeAllocationSize(NewAllocationSize))
12760 return ChangeStatus::UNCHANGED;
12761
12762 return ChangeStatus::CHANGED;
12763 }
12764
12765 /// See AbstractAttribute::manifest(...).
12766 ChangeStatus manifest(Attributor &A) override {
12767
12768 assert(isValidState() &&
12769 "Manifest should only be called if the state is valid.");
12770
12771 Instruction *I = getIRPosition().getCtxI();
12772
12773 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
12774
12775 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
12776
12777 switch (I->getOpcode()) {
12778 // TODO: add case for malloc like calls
12779 case Instruction::Alloca: {
12780
12781 AllocaInst *AI = cast<AllocaInst>(I);
12782
12783 Type *CharType = Type::getInt8Ty(I->getContext());
12784
12785 auto *NumBytesToValue =
12786 ConstantInt::get(I->getContext(), APInt(32, NumBytesToAllocate));
12787
12788 BasicBlock::iterator insertPt = AI->getIterator();
12789 insertPt = std::next(insertPt);
12790 AllocaInst *NewAllocaInst =
12791 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
12792 AI->getAlign(), AI->getName(), insertPt);
12793
12794 if (A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst))
12795 return ChangeStatus::CHANGED;
12796
12797 break;
12798 }
12799 default:
12800 break;
12801 }
12802
12803 return ChangeStatus::UNCHANGED;
12804 }
12805
12806 /// See AbstractAttribute::getAsStr().
12807 const std::string getAsStr(Attributor *A) const override {
12808 if (!isValidState())
12809 return "allocationinfo(<invalid>)";
12810 return "allocationinfo(" +
12811 (AssumedAllocatedSize == HasNoAllocationSize
12812 ? "none"
12813 : std::to_string(AssumedAllocatedSize->getFixedValue())) +
12814 ")";
12815 }
12816
12817private:
12818 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
12819
12820 // Maintain the computed allocation size of the object.
12821 // Returns (bool) weather the size of the allocation was modified or not.
12822 bool changeAllocationSize(std::optional<TypeSize> Size) {
12823 if (AssumedAllocatedSize == HasNoAllocationSize ||
12824 AssumedAllocatedSize != Size) {
12825 AssumedAllocatedSize = Size;
12826 return true;
12827 }
12828 return false;
12829 }
12830};
12831
12832struct AAAllocationInfoFloating : AAAllocationInfoImpl {
12833 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
12834 : AAAllocationInfoImpl(IRP, A) {}
12835
12836 void trackStatistics() const override {
12837 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
12838 }
12839};
12840
12841struct AAAllocationInfoReturned : AAAllocationInfoImpl {
12842 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
12843 : AAAllocationInfoImpl(IRP, A) {}
12844
12845 /// See AbstractAttribute::initialize(...).
12846 void initialize(Attributor &A) override {
12847 // TODO: we don't rewrite function argument for now because it will need to
12848 // rewrite the function signature and all call sites
12849 (void)indicatePessimisticFixpoint();
12850 }
12851
12852 void trackStatistics() const override {
12853 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
12854 }
12855};
12856
12857struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
12858 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
12859 : AAAllocationInfoImpl(IRP, A) {}
12860
12861 void trackStatistics() const override {
12862 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
12863 }
12864};
12865
12866struct AAAllocationInfoArgument : AAAllocationInfoImpl {
12867 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
12868 : AAAllocationInfoImpl(IRP, A) {}
12869
12870 void trackStatistics() const override {
12871 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
12872 }
12873};
12874
12875struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
12876 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
12877 : AAAllocationInfoImpl(IRP, A) {}
12878
12879 /// See AbstractAttribute::initialize(...).
12880 void initialize(Attributor &A) override {
12881
12882 (void)indicatePessimisticFixpoint();
12883 }
12884
12885 void trackStatistics() const override {
12886 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
12887 }
12888};
12889} // namespace
12890
12891const char AANoUnwind::ID = 0;
12892const char AANoSync::ID = 0;
12893const char AANoFree::ID = 0;
12894const char AANonNull::ID = 0;
12895const char AAMustProgress::ID = 0;
12896const char AANoRecurse::ID = 0;
12897const char AANonConvergent::ID = 0;
12898const char AAWillReturn::ID = 0;
12899const char AAUndefinedBehavior::ID = 0;
12900const char AANoAlias::ID = 0;
12901const char AAIntraFnReachability::ID = 0;
12902const char AANoReturn::ID = 0;
12903const char AAIsDead::ID = 0;
12904const char AADereferenceable::ID = 0;
12905const char AAAlign::ID = 0;
12906const char AAInstanceInfo::ID = 0;
12907const char AANoCapture::ID = 0;
12908const char AAValueSimplify::ID = 0;
12909const char AAHeapToStack::ID = 0;
12910const char AAPrivatizablePtr::ID = 0;
12911const char AAMemoryBehavior::ID = 0;
12912const char AAMemoryLocation::ID = 0;
12913const char AAValueConstantRange::ID = 0;
12914const char AAPotentialConstantValues::ID = 0;
12915const char AAPotentialValues::ID = 0;
12916const char AANoUndef::ID = 0;
12917const char AANoFPClass::ID = 0;
12918const char AACallEdges::ID = 0;
12919const char AAInterFnReachability::ID = 0;
12920const char AAPointerInfo::ID = 0;
12921const char AAAssumptionInfo::ID = 0;
12922const char AAUnderlyingObjects::ID = 0;
12923const char AAAddressSpace::ID = 0;
12924const char AAAllocationInfo::ID = 0;
12925const char AAIndirectCallInfo::ID = 0;
12926const char AAGlobalValueInfo::ID = 0;
12927const char AADenormalFPMath::ID = 0;
12928
12929// Macro magic to create the static generator function for attributes that
12930// follow the naming scheme.
12931
12932#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
12933 case IRPosition::PK: \
12934 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
12935
12936#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
12937 case IRPosition::PK: \
12938 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
12939 ++NumAAs; \
12940 break;
12941
12942#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12943 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12944 CLASS *AA = nullptr; \
12945 switch (IRP.getPositionKind()) { \
12946 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12947 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
12948 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
12949 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
12950 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
12951 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
12952 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
12953 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
12954 } \
12955 return *AA; \
12956 }
12957
12958#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12959 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12960 CLASS *AA = nullptr; \
12961 switch (IRP.getPositionKind()) { \
12962 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12963 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
12964 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
12965 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
12966 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
12967 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
12968 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
12969 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
12970 } \
12971 return *AA; \
12972 }
12973
12974#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
12975 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12976 CLASS *AA = nullptr; \
12977 switch (IRP.getPositionKind()) { \
12978 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
12979 default: \
12980 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
12981 " position!"); \
12982 } \
12983 return *AA; \
12984 }
12985
12986#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12987 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12988 CLASS *AA = nullptr; \
12989 switch (IRP.getPositionKind()) { \
12990 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12991 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
12992 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
12993 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
12994 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
12995 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
12996 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
12997 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
12998 } \
12999 return *AA; \
13000 }
13001
13002#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13003 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13004 CLASS *AA = nullptr; \
13005 switch (IRP.getPositionKind()) { \
13006 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13007 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13008 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13009 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13010 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13011 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13012 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13013 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13014 } \
13015 return *AA; \
13016 }
13017
13018#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13019 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13020 CLASS *AA = nullptr; \
13021 switch (IRP.getPositionKind()) { \
13022 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13023 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13024 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13025 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13026 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13027 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13028 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13029 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13030 } \
13031 return *AA; \
13032 }
13033
13043
13059
13064
13069
13076
13078
13079#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13080#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13081#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13082#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13083#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13084#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13085#undef SWITCH_PK_CREATE
13086#undef SWITCH_PK_INV
#define Success
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefAnalysis InstSet & ToRemove
This file contains the simple types necessary to represent the attributes associated with functions a...
#define STATS_DECLTRACK(NAME, TYPE, MSG)
static std::optional< Constant * > askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA, const IRPosition &IRP, Type &Ty)
static cl::opt< unsigned, true > MaxPotentialValues("attributor-max-potential-values", cl::Hidden, cl::desc("Maximum number of potential values to be " "tracked for each position."), cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues), cl::init(7))
static const Value * getPointerOperand(const Instruction *I, bool AllowVolatile)
Get pointer operand of memory accessing instruction.
static void clampReturnedValueStates(Attributor &A, const AAType &QueryingAA, StateType &S, const IRPosition::CallBaseContext *CBContext=nullptr)
Clamp the information known for all returned values of a function (identified by QueryingAA) into S.
#define STATS_DECLTRACK_FN_ATTR(NAME)
#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static cl::opt< int > MaxPotentialValuesIterations("attributor-max-potential-values-iterations", cl::Hidden, cl::desc("Maximum number of iterations we keep dismantling potential values."), cl::init(64))
#define STATS_DECLTRACK_CS_ATTR(NAME)
#define PIPE_OPERATOR(CLASS)
#define DefineKeys(ToTy)
static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I, bool HeaderOnly, Cycle **CPtr=nullptr)
#define STATS_DECLTRACK_ARG_ATTR(NAME)
static const Value * stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Val, const DataLayout &DL, APInt &Offset, bool GetMinOffset, bool AllowNonInbounds, bool UseAssumed=false)
#define STATS_DECLTRACK_CSRET_ATTR(NAME)
static cl::opt< bool > ManifestInternal("attributor-manifest-internal", cl::Hidden, cl::desc("Manifest Attributor internal string attributes."), cl::init(false))
static Value * constructPointer(Value *Ptr, int64_t Offset, IRBuilder< NoFolder > &IRB)
Helper function to create a pointer based on Ptr, and advanced by Offset bytes.
#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define BUILD_STAT_NAME(NAME, TYPE)
static bool isDenselyPacked(Type *Ty, const DataLayout &DL)
Checks if a type could have padding bytes.
#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static const Value * getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Ptr, int64_t &BytesOffset, const DataLayout &DL, bool AllowNonInbounds=false)
#define STATS_DECLTRACK_FNRET_ATTR(NAME)
#define STATS_DECLTRACK_CSARG_ATTR(NAME)
#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS)
#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static cl::opt< int > MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128), cl::Hidden)
#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define STATS_DECLTRACK_FLOATING_ATTR(NAME)
#define STATS_DECL(NAME, TYPE, MSG)
basic Basic Alias true
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Performs the initial survey of the specified function
Given that RA is a live value
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines DenseMapInfo traits for DenseMap.
T Content
uint64_t Size
Rewrite Partial Register Uses
#define Check(C,...)
Hexagon Common GEP
IRTranslator LLVM IR MI
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file implements a map that provides insertion order iteration.
#define T
#define T1
Metadata * LowAndHigh[]
IntegerType * Int32Ty
if(VerifyEach)
static StringRef getName(Value *V)
Basic Register Allocator
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool IsDead
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2)
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file contains some functions that are useful when dealing with strings.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringLiteral > StandardNames)
Initialize the set of available library functions based on the specified target triple.
@ Floating
This pass exposes codegen information to IR-level passes.
@ None
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
AACallGraphNode * operator*() const
A manager for alias analyses.
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are no-alias.
Class for arbitrary precision integers.
Definition: APInt.h:76
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1513
AbstractCallSite.
CallBase * getInstruction() const
Return the underlying instruction.
bool isCallbackCall() const
Return true if this ACS represents a callback call.
bool isDirectCall() const
Return true if this ACS represents a direct call.
static void getCallbackUses(const CallBase &CB, SmallVectorImpl< const Use * > &CallbackUses)
Add operand uses of CB that represent callback uses into CallbackUses.
int getCallArgOperandNo(Argument &Arg) const
Return the operand index of the underlying instruction associated with Arg.
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
Definition: Instructions.h:59
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
unsigned getAddressSpace() const
Return the address space for the allocation.
Definition: Instructions.h:112
std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
bool hasPointeeInMemoryValueAttr() const
Return true if this argument has the byval, sret, inalloca, preallocated, or byref attribute.
Definition: Function.cpp:170
bool hasReturnedAttr() const
Return true if this argument has the returned attribute.
Definition: Function.cpp:288
bool hasByValAttr() const
Return true if this argument has the byval attribute.
Definition: Function.cpp:132
const Function * getParent() const
Definition: Argument.h:40
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:46
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
iterator begin() const
Definition: ArrayRef.h:153
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:93
static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:204
FPClassTest getNoFPClass() const
Return the FPClassTest for nofpclass.
Definition: Attributes.cpp:441
Attribute::AttrKind getKindAsEnum() const
Return the attribute's kind as an enum (Attribute::AttrKind).
Definition: Attributes.cpp:320
MemoryEffects getMemoryEffects() const
Returns memory effects.
Definition: Attributes.cpp:435
static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:210
static Attribute getWithNoFPClass(LLVMContext &Context, FPClassTest Mask)
Definition: Attributes.cpp:246
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:85
static bool isEnumAttrKind(AttrKind Kind)
Definition: Attributes.h:98
static Attribute getWithMemoryEffects(LLVMContext &Context, MemoryEffects ME)
Definition: Attributes.cpp:241
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
Definition: Attributes.cpp:194
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:396
const Instruction & front() const
Definition: BasicBlock.h:452
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:198
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:164
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:155
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
BinaryOps getOpcode() const
Definition: InstrTypes.h:491
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
unsigned getNumSuccessors() const
BasicBlock * getSuccessor(unsigned i) const
Value * getCondition() const
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:66
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1623
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
Definition: InstrTypes.h:1714
Value * getCalledOperand() const
Definition: InstrTypes.h:1696
const Use & getCalledOperandUse() const
Definition: InstrTypes.h:1698
const Use & getArgOperandUse(unsigned i) const
Wrappers for getting the Use of a call argument.
Definition: InstrTypes.h:1659
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1629
bool isBundleOperand(unsigned Idx) const
Return true if the operand at index Idx is a bundle operand.
Definition: InstrTypes.h:2290
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2241
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1561
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1639
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
Definition: InstrTypes.h:1679
unsigned arg_size() const
Definition: InstrTypes.h:1646
bool isArgOperand(const Use *U) const
Definition: InstrTypes.h:1668
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:579
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:908
bool isIntegerCast() const
There are several places where we need to know if a cast instruction only deals with integer source a...
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:915
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition: InstrTypes.h:1216
bool isFalseWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:1281
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
@ ICMP_EQ
equal
Definition: InstrTypes.h:986
@ ICMP_NE
not equal
Definition: InstrTypes.h:987
bool isTrueWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:1275
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:1066
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:528
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1016
static Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
Definition: Constants.cpp:2454
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
This class represents a range of values.
Definition: ConstantRange.h:47
const APInt & getLower() const
Return the lower value for this range.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
bool isEmptySet() const
Return true if this set contains no members.
bool isSingleElement() const
Return true if this set contains exactly one member.
static ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
const APInt & getUpper() const
Return the upper value for this range.
This is an important base class in LLVM.
Definition: Constant.h:41
Analysis pass which computes a CycleInfo.
Definition: CycleAnalysis.h:47
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
iterator begin()
Definition: DenseMap.h:75
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:279
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
An instruction for ordering other memory operations.
Definition: Instructions.h:460
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
iterator_range< arg_iterator > args()
Definition: Function.h:837
const Function & getFunction() const
Definition: Function.h:160
size_t arg_size() const
Definition: Function.h:846
Argument * getArg(unsigned i) const
Definition: Function.h:831
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:669
CycleT * getCycle(const BlockT *Block) const
Find the innermost cycle containing a given block.
A possibly irreducible generalization of a Loop.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:274
bool hasLocalLinkage() const
Definition: GlobalValue.h:527
This instruction compares its operands according to the predicate given to the constructor.
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1977
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:485
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
Definition: Instruction.h:731
bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:80
const BasicBlock * getParent() const
Definition: Instruction.h:151
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:84
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
bool isTerminator() const
Definition: Instruction.h:254
bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const
Return a pointer to the next non-debug instruction in the same basic block as 'this',...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:251
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:450
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
Invoke instruction.
BasicBlock * getUnwindDest() const
BasicBlock * getNormalDest() const
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Analysis to compute lazy value information.
This pass computes, caches, and vends lazy value constraint information.
Definition: LazyValueInfo.h:31
ConstantRange getConstantRange(Value *V, Instruction *CxtI, bool UndefAllowed)
Return the ConstantRange constraint that is known to hold for the specified value at the specified in...
An instruction for reading from memory.
Definition: Instructions.h:184
Analysis pass that exposes the LoopInfo for a function.
Definition: LoopInfo.h:566
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Metadata node.
Definition: Metadata.h:1067
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1428
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1434
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
bool empty() const
Definition: MapVector.h:79
static MemoryEffectsBase readOnly()
Create MemoryEffectsBase that can read any memory.
Definition: ModRef.h:122
bool doesNotAccessMemory() const
Whether this function accesses no memory.
Definition: ModRef.h:192
static MemoryEffectsBase argMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Create MemoryEffectsBase that can only access argument memory.
Definition: ModRef.h:132
static MemoryEffectsBase inaccessibleMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Create MemoryEffectsBase that can only access inaccessible memory.
Definition: ModRef.h:138
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
Definition: ModRef.h:211
ModRefInfo getModRef(Location Loc) const
Get ModRefInfo for the given Location.
Definition: ModRef.h:165
bool onlyAccessesArgPointees() const
Whether this function only (at most) accesses argument memory.
Definition: ModRef.h:201
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
Definition: ModRef.h:195
static MemoryEffectsBase writeOnly()
Create MemoryEffectsBase that can write any memory.
Definition: ModRef.h:127
static MemoryEffectsBase inaccessibleOrArgMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Create MemoryEffectsBase that can only access inaccessible or argument memory.
Definition: ModRef.h:145
static MemoryEffectsBase none()
Create MemoryEffectsBase that cannot read or write any memory.
Definition: ModRef.h:117
bool onlyAccessesInaccessibleOrArgMem() const
Whether this function only (at most) accesses argument and inaccessible memory.
Definition: ModRef.h:217
static MemoryEffectsBase unknown()
Create MemoryEffectsBase that can read and write any memory.
Definition: ModRef.h:112
static std::optional< MemoryLocation > getOrNone(const Instruction *Inst)
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:287
Evaluate the size and offset of an object pointed to by a Value*.
static SizeOffsetValue unknown()
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents an analyzed expression in the program.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
unsigned getSmallConstantMaxTripCount(const Loop *L)
Returns the upper bound of the loop trip count as a normal unsigned value.
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:622
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
TypeSize getElementOffsetInBits(unsigned Idx) const
Definition: DataLayout.h:656
Class to represent struct types.
Definition: DerivedTypes.h:216
unsigned getNumElements() const
Random access to the elements.
Definition: DerivedTypes.h:341
Type * getElementType(unsigned N) const
Definition: DerivedTypes.h:342
Multiway switch.
Analysis pass providing the TargetTransformInfo.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool areTypesABICompatible(const Function *Caller, const Function *Callee, const ArrayRef< Type * > &Types) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:302
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition: Type.h:262
static IntegerType * getInt32Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:216
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
'undef' values are things that do not have specified contents.
Definition: Constants.h:1350
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:21
const Use & getOperandUse(unsigned i) const
Definition: User.h:182
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
Definition: User.cpp:115
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: ValueMap.h:164
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
static constexpr uint64_t MaximumAlignment
Definition: Value.h:807
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
Definition: ilist_node.h:109
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:316
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
Enumerate the SCCs of a directed graph in reverse topological order of the SCC DAG.
Definition: SCCIterator.h:49
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isAssumedReadNone(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readnone.
Definition: Attributor.cpp:654
bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readonly.
Definition: Attributor.cpp:649
raw_ostream & operator<<(raw_ostream &OS, const RangeTy &R)
Definition: Attributor.h:313
std::optional< Value * > combineOptionalValuesInAAValueLatice(const std::optional< Value * > &A, const std::optional< Value * > &B, Type *Ty)
Return the combination of A and B such that the result is a possible value of both.
Definition: Attributor.cpp:340
bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache)
Return true if the value of VAC is a valid at the position of VAC, that is a constant,...
Definition: Attributor.cpp:291
bool isAssumedThreadLocalObject(Attributor &A, Value &Obj, const AbstractAttribute &QueryingAA)
Return true if Obj is assumed to be a thread local object.
Definition: Attributor.cpp:836
bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA, const Value &V, bool ForAnalysisOnly=true)
Return true if V is dynamically unique, that is, there are no two "instances" of V at runtime with di...
Definition: Attributor.cpp:232
bool getPotentialCopiesOfStoredValue(Attributor &A, StoreInst &SI, SmallSetVector< Value *, 4 > &PotentialCopies, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values of the one stored by SI into PotentialCopies.
Definition: Attributor.cpp:600
bool isGPU(const Module &M)
Return true iff M target a GPU (and we can use GPU AS reasoning).
Definition: Attributor.cpp:201
ValueScope
Flags to distinguish intra-procedural queries from potentially inter-procedural queries.
Definition: Attributor.h:179
@ Intraprocedural
Definition: Attributor.h:180
@ Interprocedural
Definition: Attributor.h:181
bool isValidInScope(const Value &V, const Function *Scope)
Return true if V is a valid value in Scope, that is a constant or an instruction/argument of Scope.
Definition: Attributor.cpp:281
bool isPotentiallyReachable(Attributor &A, const Instruction &FromI, const Instruction &ToI, const AbstractAttribute &QueryingAA, const AA::InstExclusionSetTy *ExclusionSet=nullptr, std::function< bool(const Function &F)> GoBackwardsCB=nullptr)
Return true if ToI is potentially reachable from FromI without running into any instruction in Exclus...
Definition: Attributor.cpp:817
bool isNoSyncInst(Attributor &A, const Instruction &I, const AbstractAttribute &QueryingAA)
Return true if I is a nosync instruction.
Definition: Attributor.cpp:206
bool getPotentiallyLoadedValues(Attributor &A, LoadInst &LI, SmallSetVector< Value *, 4 > &PotentialValues, SmallSetVector< Instruction *, 4 > &PotentialValueOrigins, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values LI could read into PotentialValues.
Definition: Attributor.cpp:590
Value * getWithType(Value &V, Type &Ty)
Try to convert V to type Ty without introducing new instructions.
Definition: Attributor.cpp:317
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
AddressSpace getAddressSpace(T *V)
Definition: AVR.h:65
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Unsupported
This operation is completely unsupported on the target.
@ Undef
Value of the register doesn't matter.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
@ CE
Windows NT (Windows on ARM)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
static unsigned combineHashValue(unsigned a, unsigned b)
Simplistic combination of 32-bit hash values into 32-bit hash values.
Definition: DenseMapInfo.h:29
ElementType
The element type of an SRV or UAV resource.
Definition: DXILABI.h:68
constexpr double e
Definition: MathExtras.h:31
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:227
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:236
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
@ Length
Definition: DWP.cpp:456
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:361
bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to be non-zero when defined.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
Constant * getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty)
If this is a call to an allocation function that initializes memory to a fixed value,...
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1689
unsigned getPointerAddressSpace(const Type *T)
Definition: SPIRVUtils.h:122
auto successors(const MachineBasicBlock *BB)
bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1381
raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:2043
UseCaptureKind DetermineUseCaptureKind(const Use &U, llvm::function_ref< bool(Value *, const DataLayout &)> IsDereferenceableOrNull)
Determine what kind of capture behaviour U may exhibit.
Value * getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI)
Gets the alignment argument for an aligned_alloc-like function, using either built-in knowledge based...
Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
Definition: SCCIterator.h:233
PotentialValuesState< std::pair< AA::ValueAndContext, AA::ValueScope > > PotentialLLVMValuesState
Definition: Attributor.h:5227
bool isNoAliasCall(const Value *V)
Return true if this pointer is returned by a noalias function.
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
Definition: GraphWriter.h:359
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
@ NONE
Definition: Attributor.h:6426
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:112
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
Definition: ModRef.h:268
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:2014
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool isPointerTy(const Type *T)
Definition: SPIRVUtils.h:116
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:109
bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition: Local.cpp:418
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
Definition: SetOperations.h:23
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
Definition: ValueMapper.h:263
CallBase & promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast=nullptr)
Promote the given indirect call site to unconditionally call Callee.
bool hasAssumption(const Function &F, const KnownAssumptionString &AssumptionStr)
Return true if F has the assumption AssumptionStr attached.
Definition: Assumptions.cpp:70
RetainedKnowledge getKnowledgeFromUse(const Use *U, ArrayRef< Attribute::AttrKind > AttrKinds)
Return a valid Knowledge associated to the Use U if its Attribute kind is in AttrKinds.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Other
Any other memory.
ChangeStatus clampStateAndIndicateChange< DerefState >(DerefState &S, const DerefState &R)
Value * simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a CmpInst, fold the result or return null.
PotentialValuesState< APInt > PotentialConstantIntValuesState
Definition: Attributor.h:5225
DWARFExpression::Operation Op
bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
ChangeStatus clampStateAndIndicateChange(StateType &S, const StateType &R)
Helper function to clamp a state S of type StateType with the information in R and indicate/return if...
Definition: Attributor.h:3462
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
ChangeStatus
{
Definition: Attributor.h:483
std::optional< APInt > getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, function_ref< const Value *(const Value *)> Mapper=[](const Value *V) { return V;})
Return the size of the requested allocation.
DenseSet< StringRef > getAssumptions(const Function &F)
Return the set of all assumptions for the function F.
Definition: Assumptions.cpp:86
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
Definition: Alignment.h:111
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
@ OPTIONAL
The target may be valid if the source is not.
@ NONE
Do not track a dependence between source and target.
@ REQUIRED
The target cannot be valid if the source is not.
bool mayContainIrreducibleControl(const Function &F, const LoopInfo *LI)
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, unsigned Depth, const SimplifyQuery &SQ)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool isIdentifiedObject(const Value *V)
Return true if this pointer refers to a distinct and identifiable object.
constexpr StringRef AssumptionAttrKey
The key we use for assumption attributes.
Definition: Assumptions.h:28
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
A type to track pointer/struct usage and accesses for AAPointerInfo.
AAPointerInfo::const_bin_iterator end() const
ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I, std::optional< Value * > Content, AAPointerInfo::AccessKind Kind, Type *Ty, Instruction *RemoteI=nullptr)
Add a new Access to the state at offset Offset and with size Size.
DenseMap< const Instruction *, SmallVector< unsigned > > RemoteIMap
AAPointerInfo::const_bin_iterator begin() const
bool forallInterferingAccesses(Instruction &I, function_ref< bool(const AAPointerInfo::Access &, bool)> CB, AA::RangeTy &Range) const
See AAPointerInfo::forallInterferingAccesses.
State(State &&SIS)=default
const AAPointerInfo::Access & getAccess(unsigned Index) const
bool forallInterferingAccesses(AA::RangeTy Range, function_ref< bool(const AAPointerInfo::Access &, bool)> CB) const
See AAPointerInfo::forallInterferingAccesses.
SmallVector< AAPointerInfo::Access > AccessList
bool isAtFixpoint() const override
See AbstractState::isAtFixpoint().
static State getWorstState(const State &SIS)
Return the worst possible representable state.
AAPointerInfo::OffsetBinsTy OffsetBins
ChangeStatus indicateOptimisticFixpoint() override
See AbstractState::indicateOptimisticFixpoint().
State & operator=(const State &R)
ChangeStatus indicatePessimisticFixpoint() override
See AbstractState::indicatePessimisticFixpoint().
static State getBestState(const State &SIS)
Return the best possible representable state.
bool isValidState() const override
See AbstractState::isValidState().
----------------—AAIntraFnReachability Attribute-----------------------—
ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
unsigned Hash
Precomputed hash for this RQI.
ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To, const AA::InstExclusionSetTy *ES, bool MakeUnique)
Constructor replacement to ensure unique and stable sets are used for the cache.
An abstract interface for address space information.
Definition: Attributor.h:6237
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6276
An abstract interface for all align attributes.
Definition: Attributor.h:4270
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4301
Align getKnownAlign() const
Return known alignment.
Definition: Attributor.h:4284
static const char ID
Definition: Attributor.h:6311
An abstract attribute for getting assumption information.
Definition: Attributor.h:6163
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6191
An abstract state for querying live call edges.
Definition: Attributor.h:5487
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5530
An abstract Attribute for specializing "dynamic" components of "denormal-fp-math" and "denormal-fp-ma...
Definition: Attributor.h:6397
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6419
An abstract interface for all dereferenceable attribute.
Definition: Attributor.h:4216
uint32_t getKnownDereferenceableBytes() const
Return known dereferenceable bytes.
Definition: Attributor.h:4240
uint32_t getAssumedDereferenceableBytes() const
Return assumed dereferenceable bytes.
Definition: Attributor.h:4235
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4261
An abstract interface for llvm::GlobalValue information interference.
Definition: Attributor.h:6316
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6350
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4563
An abstract interface for indirect call information interference.
Definition: Attributor.h:6355
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6391
An abstract interface to track if a value leaves it's defining function instance.
Definition: Attributor.h:4308
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4341
An abstract Attribute for computing reachability between functions.
Definition: Attributor.h:5683
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5718
bool canReach(Attributor &A, const Function &Fn) const
If the function represented by this possition can reach Fn.
Definition: Attributor.h:5689
virtual bool instructionCanReach(Attributor &A, const Instruction &Inst, const Function &Fn, const AA::InstExclusionSetTy *ExclusionSet=nullptr) const =0
Can Inst reach Fn.
An abstract interface to determine reachability of point A to B.
Definition: Attributor.h:3814
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3842
virtual bool isAssumedReachable(Attributor &A, const Instruction &From, const Instruction &To, const AA::InstExclusionSetTy *ExclusionSet=nullptr) const =0
Returns true if 'From' instruction is assumed to reach, 'To' instruction.
An abstract interface for liveness abstract attribute.
Definition: Attributor.h:3974
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4064
An abstract interface for memory access kind related attributes (readnone/readonly/writeonly).
Definition: Attributor.h:4628
bool isAssumedReadOnly() const
Return true if we assume that the underlying value is not accessed (=written) in its respective scope...
Definition: Attributor.h:4667
bool isKnownReadNone() const
Return true if we know that the underlying value is not read or accessed in its respective scope.
Definition: Attributor.h:4655
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4694
bool isAssumedReadNone() const
Return true if we assume that the underlying value is not read or accessed in its respective scope.
Definition: Attributor.h:4659
An abstract interface for all memory location attributes (readnone/argmemonly/inaccessiblememonly/ina...
Definition: Attributor.h:4703
static std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK)
Return the locations encoded by MLK as a readable string.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4879
StateType::base_t MemoryLocationsKind
Definition: Attributor.h:4704
An abstract interface for all nonnull attributes.
Definition: Attributor.h:3590
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3625
An abstract interface for all noalias attributes.
Definition: Attributor.h:3849
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See IRAttribute::isImpliedByIR.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3888
An abstract interface for all nocapture attributes.
Definition: Attributor.h:4349
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4422
bool isAssumedNoCaptureMaybeReturned() const
Return true if we assume that the underlying value is not captured in its respective scope but we all...
Definition: Attributor.h:4403
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See IRAttribute::isImpliedByIR.
@ NO_CAPTURE_MAYBE_RETURNED
If we do not capture the value in memory or through integers we can only communicate it back as a der...
Definition: Attributor.h:4379
static void determineFunctionCaptureCapabilities(const IRPosition &IRP, const Function &F, BitIntegerState &State)
Update State according to the capture capabilities of F for position IRP.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5441
An AbstractAttribute for nofree.
Definition: Attributor.h:3895
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3938
An abstract attribute for norecurse.
Definition: Attributor.h:3683
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3707
An AbstractAttribute for noreturn.
Definition: Attributor.h:3945
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3969
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3583
static bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned)
Helper function to determine if CB is an aligned (GPU) barrier.
static bool isNonRelaxedAtomic(const Instruction *I)
Helper function used to determine whether an instruction is non-relaxed atomic.
static bool isNoSyncIntrinsic(const Instruction *I)
Helper function specific for intrinsics which are potentially volatile.
An abstract interface for all noundef attributes.
Definition: Attributor.h:5359
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5394
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See IRAttribute::isImpliedByIR.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3500
An abstract Attribute for determining the necessity of the convergent attribute.
Definition: Attributor.h:5723
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5751
bool isAssumedNotConvergent() const
Return true if "non-convergent" is assumed.
Definition: Attributor.h:5733
An abstract interface for all nonnull attributes.
Definition: Attributor.h:3632
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3676
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See AbstractAttribute::isImpliedByIR(...).
An access description.
Definition: Attributor.h:5941
A container for a list of ranges.
Definition: Attributor.h:5790
static void set_difference(const RangeList &L, const RangeList &R, RangeList &D)
Copy ranges from L that are not in R, into D.
Definition: Attributor.h:5826
An abstract interface for struct information.
Definition: Attributor.h:5755
virtual const_bin_iterator begin() const =0
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6155
virtual int64_t numOffsetBins() const =0
An abstract interface for potential values analysis.
Definition: Attributor.h:5249
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5308
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5345
static Value * getSingleValue(Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP, SmallVectorImpl< AA::ValueAndContext > &Values)
Extract the single value in Values if any.
An abstract interface for privatizability.
Definition: Attributor.h:4577
virtual std::optional< Type * > getPrivatizableType() const =0
Return the type we can choose for a private copy of the underlying value.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4619
An abstract attribute for undefined behavior.
Definition: Attributor.h:3776
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3809
An abstract attribute for getting all assumption underlying objects.
Definition: Attributor.h:6195
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6225
An abstract interface for range value analysis.
Definition: Attributor.h:4884
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4947
An abstract interface for value simplify abstract attribute.
Definition: Attributor.h:4501
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4523
An abstract attribute for willreturn.
Definition: Attributor.h:3714
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3771
Helper to represent an access offset and size, with logic to deal with uncertainty and check for over...
Definition: Attributor.h:236
static constexpr int64_t Unknown
Definition: Attributor.h:310
static RangeTy getUnknown()
Definition: Attributor.h:242
Base struct for all "concrete attribute" deductions.
Definition: Attributor.h:3282
void print(raw_ostream &OS) const
Helper functions, for debug purposes only.
Definition: Attributor.h:3366
virtual StateType & getState()=0
Return the internal abstract state for inspection.
An interface to query the internal state of an abstract attribute.
Definition: Attributor.h:2602
virtual ChangeStatus indicatePessimisticFixpoint()=0
Indicate that the abstract state should converge to the pessimistic state.
virtual bool isValidState() const =0
Return if this abstract state is in a valid state.
virtual ChangeStatus indicateOptimisticFixpoint()=0
Indicate that the abstract state should converge to the optimistic state.
Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign but the instruction.
static unsigned getHashValue(const Access &A)
static Access getTombstoneKey()
static bool isEqual(const Access &LHS, const Access &RHS)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Helper struct used in the communication between an abstract attribute (AA) that wants to change the s...
Definition: Attributor.h:2210
std::function< void(const ArgumentReplacementInfo &, AbstractCallSite, SmallVectorImpl< Value * > &)> ACSRepairCBTy
Abstract call site (ACS) repair callback type.
Definition: Attributor.h:2233
const Argument & getReplacedArg() const
Definition: Attributor.h:2240
std::function< void(const ArgumentReplacementInfo &, Function &, Function::arg_iterator)> CalleeRepairCBTy
Callee repair callback type.
Definition: Attributor.h:2219
The fixpoint analysis framework that orchestrates the attribute deduction.
Definition: Attributor.h:1507
std::function< std::optional< Value * >(const IRPosition &, const AbstractAttribute *, bool &)> SimplifictionCallbackTy
Register CB as a simplification callback.
Definition: Attributor.h:2015
Specialization of the integer state for a bit-wise encoding.
Definition: Attributor.h:2743
BitIntegerState & removeAssumedBits(base_t BitsEncoding)
Remove the bits in BitsEncoding from the "assumed bits" if not known.
Definition: Attributor.h:2768
BitIntegerState & addKnownBits(base_t Bits)
Add the bits in BitsEncoding to the "known bits".
Definition: Attributor.h:2760
Simple wrapper for a single bit (boolean) state.
Definition: Attributor.h:2886
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getDefault()
Return the assumed default mode for a function without denormal-fp-math.
static constexpr DenormalMode getInvalid()
static unsigned getHashValue(const Access &A)
static bool isEqual(const Access &LHS, const Access &RHS)
static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B)
static unsigned getHashValue(const AA::RangeTy &Range)
static ReachabilityQueryInfo< ToTy > * getEmptyKey()
static ReachabilityQueryInfo< ToTy > * getTombstoneKey()
static bool isEqual(const ReachabilityQueryInfo< ToTy > *LHS, const ReachabilityQueryInfo< ToTy > *RHS)
static unsigned getHashValue(const ReachabilityQueryInfo< ToTy > *RQI)
An information struct used to provide DenseMap with the various necessary components for a given valu...
Definition: DenseMapInfo.h:50
State for dereferenceable attribute.
Definition: Attributor.h:4070
IncIntegerState DerefBytesState
State representing for dereferenceable bytes.
Definition: Attributor.h:4086
ChangeStatus manifest(Attributor &A) override
See AbstractAttribute::manifest(...).
Definition: Attributor.h:3219
Helper to describe and deal with positions in the LLVM-IR.
Definition: Attributor.h:580
Function * getAssociatedFunction() const
Return the associated function, if any.
Definition: Attributor.h:711
static const IRPosition callsite_returned(const CallBase &CB)
Create a position describing the returned value of CB.
Definition: Attributor.h:648
static const IRPosition returned(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the returned value of F.
Definition: Attributor.h:630
Argument * getAssociatedArgument() const
Return the associated argument, if any.
Definition: Attributor.cpp:996
static const IRPosition value(const Value &V, const CallBaseContext *CBContext=nullptr)
Create a position describing the value of V.
Definition: Attributor.h:604
int getCalleeArgNo() const
Return the callee argument number of the associated value if it is an argument or call site argument,...
Definition: Attributor.h:798
static const IRPosition inst(const Instruction &I, const CallBaseContext *CBContext=nullptr)
Create a position describing the instruction I.
Definition: Attributor.h:616
static const IRPosition callsite_argument(const CallBase &CB, unsigned ArgNo)
Create a position describing the argument of CB at position ArgNo.
Definition: Attributor.h:653
@ IRP_ARGUMENT
An attribute for a function argument.
Definition: Attributor.h:594
@ IRP_RETURNED
An attribute for the function return value.
Definition: Attributor.h:590
@ IRP_CALL_SITE
An attribute for a call site (function scope).
Definition: Attributor.h:593
@ IRP_CALL_SITE_RETURNED
An attribute for a call site return value.
Definition: Attributor.h:591
@ IRP_FUNCTION
An attribute for a function (scope).
Definition: Attributor.h:592
@ IRP_CALL_SITE_ARGUMENT
An attribute for a call site argument.
Definition: Attributor.h:595
@ IRP_INVALID
An invalid position.
Definition: Attributor.h:587
Instruction * getCtxI() const
Return the context instruction, if any.
Definition: Attributor.h:764
static const IRPosition argument(const Argument &Arg, const CallBaseContext *CBContext=nullptr)
Create a position describing the argument Arg.
Definition: Attributor.h:637
Type * getAssociatedType() const
Return the type this abstract attribute is associated with.
Definition: Attributor.h:787
static const IRPosition function(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the function scope of F.
Definition: Attributor.h:623
const CallBaseContext * getCallBaseContext() const
Get the call base context from the position.
Definition: Attributor.h:926
Value & getAssociatedValue() const
Return the value this abstract attribute is associated with.
Definition: Attributor.h:778
Value & getAnchorValue() const
Return the value this abstract attribute is anchored with.
Definition: Attributor.h:697
int getCallSiteArgNo() const
Return the call site argument number of the associated value if it is an argument or call site argume...
Definition: Attributor.h:807
static const IRPosition function_scope(const IRPosition &IRP, const CallBaseContext *CBContext=nullptr)
Create a position with function scope matching the "context" of IRP.
Definition: Attributor.h:676
Kind getPositionKind() const
Return the associated position kind.
Definition: Attributor.h:876
bool isArgumentPosition() const
Return true if the position is an argument or call site argument.
Definition: Attributor.h:908
static const IRPosition callsite_function(const CallBase &CB)
Create a position describing the function scope of CB.
Definition: Attributor.h:643
Function * getAnchorScope() const
Return the Function surrounding the anchor value.
Definition: Attributor.h:752
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
Data structure to hold cached (LLVM-IR) information.
Definition: Attributor.h:1197
TargetLibraryInfo * getTargetLibraryInfoForFunction(const Function &F)
Return TargetLibraryInfo for function F.
Definition: Attributor.h:1280
bool isOnlyUsedByAssume(const Instruction &I) const
Definition: Attributor.h:1291
AP::Result * getAnalysisResultForFunction(const Function &F, bool CachedOnly=false)
Return the analysis result from a pass AP for function F.
Definition: Attributor.h:1301
State for an integer range.
Definition: Attributor.h:2928
ConstantRange getKnown() const
Return the known state encoding.
Definition: Attributor.h:2984
ConstantRange getAssumed() const
Return the assumed state encoding.
Definition: Attributor.h:2987
bool isValidState() const override
See AbstractState::isValidState() NOTE: For now we simply pretend that the worst possible state is in...
Definition: Attributor.h:2661
bool isAtFixpoint() const override
See AbstractState::isAtFixpoint()
Definition: Attributor.h:2664
ChangeStatus indicateOptimisticFixpoint() override
See AbstractState::indicateOptimisticFixpoint(...)
Definition: Attributor.h:2667
base_t getAssumed() const
Return the assumed state encoding.
Definition: Attributor.h:2682
static constexpr base_t getWorstState()
Return the worst possible representable state.
Definition: Attributor.h:2654
ChangeStatus indicatePessimisticFixpoint() override
See AbstractState::indicatePessimisticFixpoint(...)
Definition: Attributor.h:2673
Helper that allows to insert a new assumption string in the known assumption set by creating a (stati...
Definition: Assumptions.h:36
FPClassTest KnownFPClasses
Floating-point classes the value could be one of.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
A "must be executed context" for a given program point PP is the set of instructions,...
Definition: MustExecute.h:386
iterator & end()
Return an universal end iterator.
Definition: MustExecute.h:434
bool findInContextOf(const Instruction *I, const Instruction *PP)
Helper to look for I in the context of PP.
Definition: MustExecute.h:470
iterator & begin(const Instruction *PP)
Return an iterator to explore the context around PP.
Definition: MustExecute.h:420
bool checkForAllContext(const Instruction *PP, function_ref< bool(const Instruction *)> Pred)
}
Definition: MustExecute.h:456
Various options to control the behavior of getObjectSize.
A class for a set state.
Definition: Attributor.h:4958
static unsigned MaxPotentialValues
Maximum number of potential values to be tracked.
Definition: Attributor.h:5011
void unionAssumed(const MemberTy &C)
Union assumed set with the passed value.
Definition: Attributor.h:5028
static PotentialValuesState getBestState()
Return empty set as the best state of potential values.
Definition: Attributor.h:5014
const SetTy & getAssumedSet() const
Return this set.
Definition: Attributor.h:4988
Represent one information held inside an operand bundle of an llvm.assume.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
Helper to tie a abstract state implementation to an abstract attribute.
Definition: Attributor.h:3171
StateType & getState() override
See AbstractAttribute::getState(...).
Definition: Attributor.h:3179
bool unionAssumed(std::optional< Value * > Other)
Merge Other into the currently assumed simplified value.