LLVM 18.0.0git
InstCombineInternal.h
Go to the documentation of this file.
1//===- InstCombineInternal.h - InstCombine pass internals -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// This file provides internal interfaces used to implement the InstCombine.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
16#define LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
17
18#include "llvm/ADT/Statistic.h"
23#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/InstVisitor.h"
26#include "llvm/IR/Value.h"
27#include "llvm/Support/Debug.h"
31#include <cassert>
32
33#define DEBUG_TYPE "instcombine"
35
36using namespace llvm::PatternMatch;
37
38// As a default, let's assume that we want to be aggressive,
39// and attempt to traverse with no limits in attempt to sink negation.
40static constexpr unsigned NegatorDefaultMaxDepth = ~0U;
41
42// Let's guesstimate that most often we will end up visiting/producing
43// fairly small number of new instructions.
44static constexpr unsigned NegatorMaxNodesSSO = 16;
45
46namespace llvm {
47
48class AAResults;
49class APInt;
50class AssumptionCache;
52class DataLayout;
53class DominatorTree;
54class GEPOperator;
55class GlobalVariable;
56class LoopInfo;
60class User;
61
63 : public InstCombiner,
64 public InstVisitor<InstCombinerImpl, Instruction *> {
65public:
67 bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
71 const DataLayout &DL, LoopInfo *LI)
72 : InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE,
73 BFI, PSI, DL, LI) {}
74
75 virtual ~InstCombinerImpl() = default;
76
77 /// Perform early cleanup and prepare the InstCombine worklist.
78 bool prepareWorklist(Function &F,
80
81 /// Run the combiner over the entire worklist until it is empty.
82 ///
83 /// \returns true if the IR is changed.
84 bool run();
85
86 // Visitation implementation - Implement instruction combining for different
87 // instruction types. The semantics are as follows:
88 // Return Value:
89 // null - No change was made
90 // I - Change was made, I is still valid, I may be dead though
91 // otherwise - Change was made, replace I with returned instruction
92 //
93 Instruction *visitFNeg(UnaryOperator &I);
94 Instruction *visitAdd(BinaryOperator &I);
95 Instruction *visitFAdd(BinaryOperator &I);
96 Value *OptimizePointerDifference(
97 Value *LHS, Value *RHS, Type *Ty, bool isNUW);
98 Instruction *visitSub(BinaryOperator &I);
99 Instruction *visitFSub(BinaryOperator &I);
100 Instruction *visitMul(BinaryOperator &I);
101 Instruction *foldFMulReassoc(BinaryOperator &I);
102 Instruction *visitFMul(BinaryOperator &I);
103 Instruction *visitURem(BinaryOperator &I);
104 Instruction *visitSRem(BinaryOperator &I);
105 Instruction *visitFRem(BinaryOperator &I);
106 bool simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I);
107 Instruction *commonIRemTransforms(BinaryOperator &I);
108 Instruction *commonIDivTransforms(BinaryOperator &I);
109 Instruction *visitUDiv(BinaryOperator &I);
110 Instruction *visitSDiv(BinaryOperator &I);
111 Instruction *visitFDiv(BinaryOperator &I);
112 Value *simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1, bool Inverted);
113 Instruction *visitAnd(BinaryOperator &I);
114 Instruction *visitOr(BinaryOperator &I);
115 bool sinkNotIntoLogicalOp(Instruction &I);
116 bool sinkNotIntoOtherHandOfLogicalOp(Instruction &I);
117 Instruction *visitXor(BinaryOperator &I);
118 Instruction *visitShl(BinaryOperator &I);
119 Value *reassociateShiftAmtsOfTwoSameDirectionShifts(
120 BinaryOperator *Sh0, const SimplifyQuery &SQ,
121 bool AnalyzeForSignBitExtraction = false);
122 Instruction *canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(
124 Instruction *foldVariableSignZeroExtensionOfVariableHighBitExtract(
125 BinaryOperator &OldAShr);
126 Instruction *visitAShr(BinaryOperator &I);
127 Instruction *visitLShr(BinaryOperator &I);
128 Instruction *commonShiftTransforms(BinaryOperator &I);
129 Instruction *visitFCmpInst(FCmpInst &I);
130 CmpInst *canonicalizeICmpPredicate(CmpInst &I);
131 Instruction *visitICmpInst(ICmpInst &I);
132 Instruction *FoldShiftByConstant(Value *Op0, Constant *Op1,
134 Instruction *commonCastTransforms(CastInst &CI);
135 Instruction *visitTrunc(TruncInst &CI);
136 Instruction *visitZExt(ZExtInst &Zext);
137 Instruction *visitSExt(SExtInst &Sext);
138 Instruction *visitFPTrunc(FPTruncInst &CI);
139 Instruction *visitFPExt(CastInst &CI);
140 Instruction *visitFPToUI(FPToUIInst &FI);
141 Instruction *visitFPToSI(FPToSIInst &FI);
142 Instruction *visitUIToFP(CastInst &CI);
143 Instruction *visitSIToFP(CastInst &CI);
144 Instruction *visitPtrToInt(PtrToIntInst &CI);
145 Instruction *visitIntToPtr(IntToPtrInst &CI);
146 Instruction *visitBitCast(BitCastInst &CI);
147 Instruction *visitAddrSpaceCast(AddrSpaceCastInst &CI);
148 Instruction *foldItoFPtoI(CastInst &FI);
150 Instruction *visitCallInst(CallInst &CI);
151 Instruction *visitInvokeInst(InvokeInst &II);
152 Instruction *visitCallBrInst(CallBrInst &CBI);
153
154 Instruction *SliceUpIllegalIntegerPHI(PHINode &PN);
155 Instruction *visitPHINode(PHINode &PN);
156 Instruction *visitGetElementPtrInst(GetElementPtrInst &GEP);
157 Instruction *visitGEPOfGEP(GetElementPtrInst &GEP, GEPOperator *Src);
158 Instruction *visitAllocaInst(AllocaInst &AI);
159 Instruction *visitAllocSite(Instruction &FI);
160 Instruction *visitFree(CallInst &FI, Value *FreedOp);
161 Instruction *visitLoadInst(LoadInst &LI);
162 Instruction *visitStoreInst(StoreInst &SI);
163 Instruction *visitAtomicRMWInst(AtomicRMWInst &SI);
164 Instruction *visitUnconditionalBranchInst(BranchInst &BI);
165 Instruction *visitBranchInst(BranchInst &BI);
166 Instruction *visitFenceInst(FenceInst &FI);
167 Instruction *visitSwitchInst(SwitchInst &SI);
168 Instruction *visitReturnInst(ReturnInst &RI);
169 Instruction *visitUnreachableInst(UnreachableInst &I);
171 foldAggregateConstructionIntoAggregateReuse(InsertValueInst &OrigIVI);
172 Instruction *visitInsertValueInst(InsertValueInst &IV);
173 Instruction *visitInsertElementInst(InsertElementInst &IE);
174 Instruction *visitExtractElementInst(ExtractElementInst &EI);
175 Instruction *simplifyBinOpSplats(ShuffleVectorInst &SVI);
176 Instruction *visitShuffleVectorInst(ShuffleVectorInst &SVI);
177 Instruction *visitExtractValueInst(ExtractValueInst &EV);
178 Instruction *visitLandingPadInst(LandingPadInst &LI);
179 Instruction *visitVAEndInst(VAEndInst &I);
180 Value *pushFreezeToPreventPoisonFromPropagating(FreezeInst &FI);
181 bool freezeOtherUses(FreezeInst &FI);
182 Instruction *foldFreezeIntoRecurrence(FreezeInst &I, PHINode *PN);
183 Instruction *visitFreeze(FreezeInst &I);
184
185 /// Specify what to return for unhandled instructions.
187
188 /// True when DB dominates all uses of DI except UI.
189 /// UI must be in the same block as DI.
190 /// The routine checks that the DI parent and DB are different.
191 bool dominatesAllUses(const Instruction *DI, const Instruction *UI,
192 const BasicBlock *DB) const;
193
194 /// Try to replace select with select operand SIOpd in SI-ICmp sequence.
195 bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp,
196 const unsigned SIOpd);
197
198 LoadInst *combineLoadToNewType(LoadInst &LI, Type *NewTy,
199 const Twine &Suffix = "");
200
202 FPClassTest Interested = fcAllFlags,
203 const Instruction *CtxI = nullptr,
204 unsigned Depth = 0) const {
205 return llvm::computeKnownFPClass(Val, FMF, DL, Interested, Depth, &TLI, &AC,
206 CtxI, &DT);
207 }
208
210 FPClassTest Interested = fcAllFlags,
211 const Instruction *CtxI = nullptr,
212 unsigned Depth = 0) const {
213 return llvm::computeKnownFPClass(Val, DL, Interested, Depth, &TLI, &AC,
214 CtxI, &DT);
215 }
216
217 /// Check if fmul \p MulVal, +0.0 will yield +0.0 (or signed zero is
218 /// ignorable).
220 const Instruction *CtxI) const;
221
222 Constant *getLosslessTrunc(Constant *C, Type *TruncTy, unsigned ExtOp) {
223 Constant *TruncC = ConstantExpr::getTrunc(C, TruncTy);
224 Constant *ExtTruncC =
225 ConstantFoldCastOperand(ExtOp, TruncC, C->getType(), DL);
226 if (ExtTruncC && ExtTruncC == C)
227 return TruncC;
228 return nullptr;
229 }
230
232 return getLosslessTrunc(C, TruncTy, Instruction::ZExt);
233 }
234
236 return getLosslessTrunc(C, TruncTy, Instruction::SExt);
237 }
238
239private:
240 bool annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI);
241 bool isDesirableIntType(unsigned BitWidth) const;
242 bool shouldChangeType(unsigned FromBitWidth, unsigned ToBitWidth) const;
243 bool shouldChangeType(Type *From, Type *To) const;
244 Value *dyn_castNegVal(Value *V) const;
245
246 /// Classify whether a cast is worth optimizing.
247 ///
248 /// This is a helper to decide whether the simplification of
249 /// logic(cast(A), cast(B)) to cast(logic(A, B)) should be performed.
250 ///
251 /// \param CI The cast we are interested in.
252 ///
253 /// \return true if this cast actually results in any code being generated and
254 /// if it cannot already be eliminated by some other transformation.
255 bool shouldOptimizeCast(CastInst *CI);
256
257 /// Try to optimize a sequence of instructions checking if an operation
258 /// on LHS and RHS overflows.
259 ///
260 /// If this overflow check is done via one of the overflow check intrinsics,
261 /// then CtxI has to be the call instruction calling that intrinsic. If this
262 /// overflow check is done by arithmetic followed by a compare, then CtxI has
263 /// to be the arithmetic instruction.
264 ///
265 /// If a simplification is possible, stores the simplified result of the
266 /// operation in OperationResult and result of the overflow check in
267 /// OverflowResult, and return true. If no simplification is possible,
268 /// returns false.
269 bool OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, bool IsSigned,
270 Value *LHS, Value *RHS,
271 Instruction &CtxI, Value *&OperationResult,
273
274 Instruction *visitCallBase(CallBase &Call);
275 Instruction *tryOptimizeCall(CallInst *CI);
276 bool transformConstExprCastCall(CallBase &Call);
277 Instruction *transformCallThroughTrampoline(CallBase &Call,
278 IntrinsicInst &Tramp);
279
280 Value *simplifyMaskedLoad(IntrinsicInst &II);
281 Instruction *simplifyMaskedStore(IntrinsicInst &II);
282 Instruction *simplifyMaskedGather(IntrinsicInst &II);
283 Instruction *simplifyMaskedScatter(IntrinsicInst &II);
284
285 /// Transform (zext icmp) to bitwise / integer operations in order to
286 /// eliminate it.
287 ///
288 /// \param ICI The icmp of the (zext icmp) pair we are interested in.
289 /// \parem CI The zext of the (zext icmp) pair we are interested in.
290 ///
291 /// \return null if the transformation cannot be performed. If the
292 /// transformation can be performed the new instruction that replaces the
293 /// (zext icmp) pair will be returned.
294 Instruction *transformZExtICmp(ICmpInst *Cmp, ZExtInst &Zext);
295
296 Instruction *transformSExtICmp(ICmpInst *Cmp, SExtInst &Sext);
297
298 bool willNotOverflowSignedAdd(const WithCache<const Value *> &LHS,
300 const Instruction &CxtI) const {
301 return computeOverflowForSignedAdd(LHS, RHS, &CxtI) ==
302 OverflowResult::NeverOverflows;
303 }
304
305 bool willNotOverflowUnsignedAdd(const WithCache<const Value *> &LHS,
307 const Instruction &CxtI) const {
308 return computeOverflowForUnsignedAdd(LHS, RHS, &CxtI) ==
309 OverflowResult::NeverOverflows;
310 }
311
312 bool willNotOverflowAdd(const Value *LHS, const Value *RHS,
313 const Instruction &CxtI, bool IsSigned) const {
314 return IsSigned ? willNotOverflowSignedAdd(LHS, RHS, CxtI)
315 : willNotOverflowUnsignedAdd(LHS, RHS, CxtI);
316 }
317
318 bool willNotOverflowSignedSub(const Value *LHS, const Value *RHS,
319 const Instruction &CxtI) const {
320 return computeOverflowForSignedSub(LHS, RHS, &CxtI) ==
321 OverflowResult::NeverOverflows;
322 }
323
324 bool willNotOverflowUnsignedSub(const Value *LHS, const Value *RHS,
325 const Instruction &CxtI) const {
326 return computeOverflowForUnsignedSub(LHS, RHS, &CxtI) ==
327 OverflowResult::NeverOverflows;
328 }
329
330 bool willNotOverflowSub(const Value *LHS, const Value *RHS,
331 const Instruction &CxtI, bool IsSigned) const {
332 return IsSigned ? willNotOverflowSignedSub(LHS, RHS, CxtI)
333 : willNotOverflowUnsignedSub(LHS, RHS, CxtI);
334 }
335
336 bool willNotOverflowSignedMul(const Value *LHS, const Value *RHS,
337 const Instruction &CxtI) const {
338 return computeOverflowForSignedMul(LHS, RHS, &CxtI) ==
339 OverflowResult::NeverOverflows;
340 }
341
342 bool willNotOverflowUnsignedMul(const Value *LHS, const Value *RHS,
343 const Instruction &CxtI) const {
344 return computeOverflowForUnsignedMul(LHS, RHS, &CxtI) ==
345 OverflowResult::NeverOverflows;
346 }
347
348 bool willNotOverflowMul(const Value *LHS, const Value *RHS,
349 const Instruction &CxtI, bool IsSigned) const {
350 return IsSigned ? willNotOverflowSignedMul(LHS, RHS, CxtI)
351 : willNotOverflowUnsignedMul(LHS, RHS, CxtI);
352 }
353
355 const Value *RHS, const Instruction &CxtI,
356 bool IsSigned) const {
357 switch (Opcode) {
358 case Instruction::Add: return willNotOverflowAdd(LHS, RHS, CxtI, IsSigned);
359 case Instruction::Sub: return willNotOverflowSub(LHS, RHS, CxtI, IsSigned);
360 case Instruction::Mul: return willNotOverflowMul(LHS, RHS, CxtI, IsSigned);
361 default: llvm_unreachable("Unexpected opcode for overflow query");
362 }
363 }
364
365 Value *EmitGEPOffset(User *GEP);
366 Instruction *scalarizePHI(ExtractElementInst &EI, PHINode *PN);
367 Instruction *foldBitcastExtElt(ExtractElementInst &ExtElt);
368 Instruction *foldCastedBitwiseLogic(BinaryOperator &I);
369 Instruction *foldBinopOfSextBoolToSelect(BinaryOperator &I);
370 Instruction *narrowBinOp(TruncInst &Trunc);
371 Instruction *narrowMaskedBinOp(BinaryOperator &And);
372 Instruction *narrowMathIfNoOverflow(BinaryOperator &I);
373 Instruction *narrowFunnelShift(TruncInst &Trunc);
374 Instruction *optimizeBitCastFromPhi(CastInst &CI, PHINode *PN);
375 Instruction *matchSAddSubSat(IntrinsicInst &MinMax1);
376 Instruction *foldNot(BinaryOperator &I);
377 Instruction *foldBinOpOfDisplacedShifts(BinaryOperator &I);
378
379 /// Determine if a pair of casts can be replaced by a single cast.
380 ///
381 /// \param CI1 The first of a pair of casts.
382 /// \param CI2 The second of a pair of casts.
383 ///
384 /// \return 0 if the cast pair cannot be eliminated, otherwise returns an
385 /// Instruction::CastOps value for a cast that can replace the pair, casting
386 /// CI1->getSrcTy() to CI2->getDstTy().
387 ///
388 /// \see CastInst::isEliminableCastPair
389 Instruction::CastOps isEliminableCastPair(const CastInst *CI1,
390 const CastInst *CI2);
391 Value *simplifyIntToPtrRoundTripCast(Value *Val);
392
393 Value *foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, Instruction &I,
394 bool IsAnd, bool IsLogical = false);
395 Value *foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS, BinaryOperator &Xor);
396
397 Value *foldEqOfParts(ICmpInst *Cmp0, ICmpInst *Cmp1, bool IsAnd);
398
399 Value *foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1, ICmpInst *ICmp2,
400 bool IsAnd);
401
402 /// Optimize (fcmp)&(fcmp) or (fcmp)|(fcmp).
403 /// NOTE: Unlike most of instcombine, this returns a Value which should
404 /// already be inserted into the function.
405 Value *foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS, bool IsAnd,
406 bool IsLogicalSelect = false);
407
408 Instruction *foldLogicOfIsFPClass(BinaryOperator &Operator, Value *LHS,
409 Value *RHS);
410
412 canonicalizeConditionalNegationViaMathToSelect(BinaryOperator &i);
413
414 Value *foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS, ICmpInst *RHS,
415 Instruction *CxtI, bool IsAnd,
416 bool IsLogical = false);
417 Value *matchSelectFromAndOr(Value *A, Value *B, Value *C, Value *D,
418 bool InvertFalseVal = false);
419 Value *getSelectCondition(Value *A, Value *B, bool ABIsTheSame);
420
421 Instruction *foldLShrOverflowBit(BinaryOperator &I);
422 Instruction *foldExtractOfOverflowIntrinsic(ExtractValueInst &EV);
423 Instruction *foldIntrinsicWithOverflowCommon(IntrinsicInst *II);
424 Instruction *foldIntrinsicIsFPClass(IntrinsicInst &II);
425 Instruction *foldFPSignBitOps(BinaryOperator &I);
426 Instruction *foldFDivConstantDivisor(BinaryOperator &I);
427
428 // Optimize one of these forms:
429 // and i1 Op, SI / select i1 Op, i1 SI, i1 false (if IsAnd = true)
430 // or i1 Op, SI / select i1 Op, i1 true, i1 SI (if IsAnd = false)
431 // into simplier select instruction using isImpliedCondition.
432 Instruction *foldAndOrOfSelectUsingImpliedCond(Value *Op, SelectInst &SI,
433 bool IsAnd);
434
435 Instruction *hoistFNegAboveFMulFDiv(Value *FNegOp, Instruction &FMFSource);
436
437public:
438 /// Create and insert the idiom we use to indicate a block is unreachable
439 /// without having to rewrite the CFG from within InstCombine.
441 auto &Ctx = InsertAt->getContext();
442 auto *SI = new StoreInst(ConstantInt::getTrue(Ctx),
443 PoisonValue::get(PointerType::getUnqual(Ctx)),
444 /*isVolatile*/ false, Align(1));
445 InsertNewInstBefore(SI, InsertAt->getIterator());
446 }
447
448 /// Combiner aware instruction erasure.
449 ///
450 /// When dealing with an instruction that has side effects or produces a void
451 /// value, we can't rely on DCE to delete the instruction. Instead, visit
452 /// methods should return the value returned by this function.
454 LLVM_DEBUG(dbgs() << "IC: ERASE " << I << '\n');
455 assert(I.use_empty() && "Cannot erase instruction that is used!");
457
458 // Make sure that we reprocess all operands now that we reduced their
459 // use counts.
460 SmallVector<Value *> Ops(I.operands());
461 Worklist.remove(&I);
462 I.eraseFromParent();
463 for (Value *Op : Ops)
464 Worklist.handleUseCountDecrement(Op);
465 MadeIRChange = true;
466 return nullptr; // Don't do anything with FI
467 }
468
469 OverflowResult computeOverflow(
470 Instruction::BinaryOps BinaryOp, bool IsSigned,
471 Value *LHS, Value *RHS, Instruction *CxtI) const;
472
473 /// Performs a few simplifications for operators which are associative
474 /// or commutative.
475 bool SimplifyAssociativeOrCommutative(BinaryOperator &I);
476
477 /// Tries to simplify binary operations which some other binary
478 /// operation distributes over.
479 ///
480 /// It does this by either by factorizing out common terms (eg "(A*B)+(A*C)"
481 /// -> "A*(B+C)") or expanding out if this results in simplifications (eg: "A
482 /// & (B | C) -> (A&B) | (A&C)" if this is a win). Returns the simplified
483 /// value, or null if it didn't simplify.
484 Value *foldUsingDistributiveLaws(BinaryOperator &I);
485
486 /// Tries to simplify add operations using the definition of remainder.
487 ///
488 /// The definition of remainder is X % C = X - (X / C ) * C. The add
489 /// expression X % C0 + (( X / C0 ) % C1) * C0 can be simplified to
490 /// X % (C0 * C1)
491 Value *SimplifyAddWithRemainder(BinaryOperator &I);
492
493 // Binary Op helper for select operations where the expression can be
494 // efficiently reorganized.
495 Value *SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS,
496 Value *RHS);
497
498 // (Binop1 (Binop2 (logic_shift X, C), C1), (logic_shift Y, C))
499 // -> (logic_shift (Binop1 (Binop2 X, inv_logic_shift(C1, C)), Y), C)
500 // (Binop1 (Binop2 (logic_shift X, Amt), Mask), (logic_shift Y, Amt))
501 // -> (BinOp (logic_shift (BinOp X, Y)), Mask)
502 Instruction *foldBinOpShiftWithShift(BinaryOperator &I);
503
504 /// Tries to simplify binops of select and cast of the select condition.
505 ///
506 /// (Binop (cast C), (select C, T, F))
507 /// -> (select C, C0, C1)
508 Instruction *foldBinOpOfSelectAndCastOfSelectCondition(BinaryOperator &I);
509
510 /// This tries to simplify binary operations by factorizing out common terms
511 /// (e. g. "(A*B)+(A*C)" -> "A*(B+C)").
512 Value *tryFactorizationFolds(BinaryOperator &I);
513
514 /// Match a select chain which produces one of three values based on whether
515 /// the LHS is less than, equal to, or greater than RHS respectively.
516 /// Return true if we matched a three way compare idiom. The LHS, RHS, Less,
517 /// Equal and Greater values are saved in the matching process and returned to
518 /// the caller.
519 bool matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, Value *&RHS,
520 ConstantInt *&Less, ConstantInt *&Equal,
521 ConstantInt *&Greater);
522
523 /// Attempts to replace V with a simpler value based on the demanded
524 /// bits.
525 Value *SimplifyDemandedUseBits(Value *V, APInt DemandedMask, KnownBits &Known,
526 unsigned Depth, Instruction *CxtI);
527 bool SimplifyDemandedBits(Instruction *I, unsigned Op,
528 const APInt &DemandedMask, KnownBits &Known,
529 unsigned Depth = 0) override;
530
531 /// Helper routine of SimplifyDemandedUseBits. It computes KnownZero/KnownOne
532 /// bits. It also tries to handle simplifications that can be done based on
533 /// DemandedMask, but without modifying the Instruction.
534 Value *SimplifyMultipleUseDemandedBits(Instruction *I,
535 const APInt &DemandedMask,
536 KnownBits &Known,
537 unsigned Depth, Instruction *CxtI);
538
539 /// Helper routine of SimplifyDemandedUseBits. It tries to simplify demanded
540 /// bit for "r1 = shr x, c1; r2 = shl r1, c2" instruction sequence.
541 Value *simplifyShrShlDemandedBits(
542 Instruction *Shr, const APInt &ShrOp1, Instruction *Shl,
543 const APInt &ShlOp1, const APInt &DemandedMask, KnownBits &Known);
544
545 /// Tries to simplify operands to an integer instruction based on its
546 /// demanded bits.
547 bool SimplifyDemandedInstructionBits(Instruction &Inst);
548 bool SimplifyDemandedInstructionBits(Instruction &Inst, KnownBits &Known);
549
550 Value *SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
551 APInt &UndefElts, unsigned Depth = 0,
552 bool AllowMultipleUsers = false) override;
553
554 /// Canonicalize the position of binops relative to shufflevector.
555 Instruction *foldVectorBinop(BinaryOperator &Inst);
557 Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf);
558
559 /// Given a binary operator, cast instruction, or select which has a PHI node
560 /// as operand #0, see if we can fold the instruction into the PHI (which is
561 /// only possible if all operands to the PHI are constants).
562 Instruction *foldOpIntoPhi(Instruction &I, PHINode *PN);
563
564 /// For a binary operator with 2 phi operands, try to hoist the binary
565 /// operation before the phi. This can result in fewer instructions in
566 /// patterns where at least one set of phi operands simplifies.
567 /// Example:
568 /// BB3: binop (phi [X, BB1], [C1, BB2]), (phi [Y, BB1], [C2, BB2])
569 /// -->
570 /// BB1: BO = binop X, Y
571 /// BB3: phi [BO, BB1], [(binop C1, C2), BB2]
572 Instruction *foldBinopWithPhiOperands(BinaryOperator &BO);
573
574 /// Given an instruction with a select as one operand and a constant as the
575 /// other operand, try to fold the binary operator into the select arguments.
576 /// This also works for Cast instructions, which obviously do not have a
577 /// second operand.
578 Instruction *FoldOpIntoSelect(Instruction &Op, SelectInst *SI,
579 bool FoldWithMultiUse = false);
580
581 /// This is a convenience wrapper function for the above two functions.
582 Instruction *foldBinOpIntoSelectOrPhi(BinaryOperator &I);
583
584 Instruction *foldAddWithConstant(BinaryOperator &Add);
585
586 Instruction *foldSquareSumInt(BinaryOperator &I);
587 Instruction *foldSquareSumFP(BinaryOperator &I);
588
589 /// Try to rotate an operation below a PHI node, using PHI nodes for
590 /// its operands.
591 Instruction *foldPHIArgOpIntoPHI(PHINode &PN);
592 Instruction *foldPHIArgBinOpIntoPHI(PHINode &PN);
593 Instruction *foldPHIArgInsertValueInstructionIntoPHI(PHINode &PN);
594 Instruction *foldPHIArgExtractValueInstructionIntoPHI(PHINode &PN);
595 Instruction *foldPHIArgGEPIntoPHI(PHINode &PN);
596 Instruction *foldPHIArgLoadIntoPHI(PHINode &PN);
597 Instruction *foldPHIArgZextsIntoPHI(PHINode &PN);
598 Instruction *foldPHIArgIntToPtrToPHI(PHINode &PN);
599
600 /// If an integer typed PHI has only one use which is an IntToPtr operation,
601 /// replace the PHI with an existing pointer typed PHI if it exists. Otherwise
602 /// insert a new pointer typed PHI and replace the original one.
603 bool foldIntegerTypedPHI(PHINode &PN);
604
605 /// Helper function for FoldPHIArgXIntoPHI() to set debug location for the
606 /// folded operation.
607 void PHIArgMergedDebugLoc(Instruction *Inst, PHINode &PN);
608
609 Instruction *foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
611 Instruction *foldSelectICmp(ICmpInst::Predicate Pred, SelectInst *SI,
612 Value *RHS, const ICmpInst &I);
613 bool foldAllocaCmp(AllocaInst *Alloca);
614 Instruction *foldCmpLoadFromIndexedGlobal(LoadInst *LI,
616 GlobalVariable *GV, CmpInst &ICI,
617 ConstantInt *AndCst = nullptr);
618 Instruction *foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI,
619 Constant *RHSC);
620 Instruction *foldICmpAddOpConst(Value *X, const APInt &C,
622 Instruction *foldICmpWithCastOp(ICmpInst &ICmp);
623 Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp);
624
625 Instruction *foldICmpUsingKnownBits(ICmpInst &Cmp);
627 Instruction *foldICmpWithConstant(ICmpInst &Cmp);
628 Instruction *foldICmpUsingBoolRange(ICmpInst &I);
629 Instruction *foldICmpInstWithConstant(ICmpInst &Cmp);
630 Instruction *foldICmpInstWithConstantNotInt(ICmpInst &Cmp);
631 Instruction *foldICmpInstWithConstantAllowUndef(ICmpInst &Cmp,
632 const APInt &C);
633 Instruction *foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ);
634 Instruction *foldICmpWithMinMaxImpl(Instruction &I, MinMaxIntrinsic *MinMax,
635 Value *Z, ICmpInst::Predicate Pred);
636 Instruction *foldICmpWithMinMax(ICmpInst &Cmp);
637 Instruction *foldICmpEquality(ICmpInst &Cmp);
638 Instruction *foldIRemByPowerOfTwoToBitTest(ICmpInst &I);
639 Instruction *foldSignBitTest(ICmpInst &I);
640 Instruction *foldICmpWithZero(ICmpInst &Cmp);
641
642 Value *foldMultiplicationOverflowCheck(ICmpInst &Cmp);
643
644 Instruction *foldICmpBinOpWithConstant(ICmpInst &Cmp, BinaryOperator *BO,
645 const APInt &C);
646 Instruction *foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select,
647 ConstantInt *C);
648 Instruction *foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc,
649 const APInt &C);
650 Instruction *foldICmpTruncWithTruncOrExt(ICmpInst &Cmp,
651 const SimplifyQuery &Q);
652 Instruction *foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And,
653 const APInt &C);
654 Instruction *foldICmpXorConstant(ICmpInst &Cmp, BinaryOperator *Xor,
655 const APInt &C);
656 Instruction *foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or,
657 const APInt &C);
658 Instruction *foldICmpMulConstant(ICmpInst &Cmp, BinaryOperator *Mul,
659 const APInt &C);
660 Instruction *foldICmpShlConstant(ICmpInst &Cmp, BinaryOperator *Shl,
661 const APInt &C);
662 Instruction *foldICmpShrConstant(ICmpInst &Cmp, BinaryOperator *Shr,
663 const APInt &C);
664 Instruction *foldICmpSRemConstant(ICmpInst &Cmp, BinaryOperator *UDiv,
665 const APInt &C);
666 Instruction *foldICmpUDivConstant(ICmpInst &Cmp, BinaryOperator *UDiv,
667 const APInt &C);
668 Instruction *foldICmpDivConstant(ICmpInst &Cmp, BinaryOperator *Div,
669 const APInt &C);
670 Instruction *foldICmpSubConstant(ICmpInst &Cmp, BinaryOperator *Sub,
671 const APInt &C);
672 Instruction *foldICmpAddConstant(ICmpInst &Cmp, BinaryOperator *Add,
673 const APInt &C);
674 Instruction *foldICmpAndConstConst(ICmpInst &Cmp, BinaryOperator *And,
675 const APInt &C1);
676 Instruction *foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And,
677 const APInt &C1, const APInt &C2);
678 Instruction *foldICmpXorShiftConst(ICmpInst &Cmp, BinaryOperator *Xor,
679 const APInt &C);
680 Instruction *foldICmpShrConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1,
681 const APInt &C2);
682 Instruction *foldICmpShlConstConst(ICmpInst &I, Value *ShAmt, const APInt &C1,
683 const APInt &C2);
684
685 Instruction *foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp,
686 BinaryOperator *BO,
687 const APInt &C);
688 Instruction *foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II,
689 const APInt &C);
690 Instruction *foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II,
691 const APInt &C);
692 Instruction *foldICmpBitCast(ICmpInst &Cmp);
693 Instruction *foldICmpWithTrunc(ICmpInst &Cmp);
694
695 // Helpers of visitSelectInst().
698 Instruction *foldSelectOpOp(SelectInst &SI, Instruction *TI, Instruction *FI);
699 Instruction *foldSelectIntoOp(SelectInst &SI, Value *, Value *);
701 Value *A, Value *B, Instruction &Outer,
706 unsigned Depth = 0);
707
708 Value *insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi,
709 bool isSigned, bool Inside);
710 bool mergeStoreIntoSuccessor(StoreInst &SI);
711
712 /// Given an initial instruction, check to see if it is the root of a
713 /// bswap/bitreverse idiom. If so, return the equivalent bswap/bitreverse
714 /// intrinsic.
715 Instruction *matchBSwapOrBitReverse(Instruction &I, bool MatchBSwaps,
716 bool MatchBitReversals);
717
718 Instruction *SimplifyAnyMemTransfer(AnyMemTransferInst *MI);
719 Instruction *SimplifyAnyMemSet(AnyMemSetInst *MI);
720
721 Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
722
723 bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock);
724
725 bool removeInstructionsBeforeUnreachable(Instruction &I);
726 void addDeadEdge(BasicBlock *From, BasicBlock *To,
728 void handleUnreachableFrom(Instruction *I,
730 void handlePotentiallyDeadBlocks(SmallVectorImpl<BasicBlock *> &Worklist);
731 void handlePotentiallyDeadSuccessors(BasicBlock *BB, BasicBlock *LiveSucc);
732 void freelyInvertAllUsersOf(Value *V, Value *IgnoredUser = nullptr);
733};
734
735class Negator final {
736 /// Top-to-bottom, def-to-use negated instruction tree we produced.
738
740 BuilderTy Builder;
741
742 const SimplifyQuery &SQ;
743
744 const bool IsTrulyNegation;
745
746 SmallDenseMap<Value *, Value *> NegationsCache;
747
748 Negator(LLVMContext &C, const SimplifyQuery &SQ, bool IsTrulyNegation);
749
750#if LLVM_ENABLE_STATS
751 unsigned NumValuesVisitedInThisNegator = 0;
752 ~Negator();
753#endif
754
755 using Result = std::pair<ArrayRef<Instruction *> /*NewInstructions*/,
756 Value * /*NegatedRoot*/>;
757
758 std::array<Value *, 2> getSortedOperandsOfBinOp(Instruction *I);
759
760 [[nodiscard]] Value *visitImpl(Value *V, bool IsNSW, unsigned Depth);
761
762 [[nodiscard]] Value *negate(Value *V, bool IsNSW, unsigned Depth);
763
764 /// Recurse depth-first and attempt to sink the negation.
765 /// FIXME: use worklist?
766 [[nodiscard]] std::optional<Result> run(Value *Root, bool IsNSW);
767
768 Negator(const Negator &) = delete;
769 Negator(Negator &&) = delete;
770 Negator &operator=(const Negator &) = delete;
771 Negator &operator=(Negator &&) = delete;
772
773public:
774 /// Attempt to negate \p Root. Retuns nullptr if negation can't be performed,
775 /// otherwise returns negated value.
776 [[nodiscard]] static Value *Negate(bool LHSIsZero, bool IsNSW, Value *Root,
777 InstCombinerImpl &IC);
778};
779
780} // end namespace llvm
781
782#undef DEBUG_TYPE
783
784#endif // LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static bool foldICmpWithDominatingICmp(CmpInst *Cmp, const TargetLowering &TLI)
For pattern like:
#define LLVM_LIBRARY_VISIBILITY
Definition: Compiler.h:131
static bool willNotOverflow(BinaryOpIntrinsic *BO, LazyValueInfo *LVI)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Align
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Hexagon Common GEP
IRTranslator LLVM IR MI
static constexpr unsigned NegatorMaxNodesSSO
static constexpr unsigned NegatorDefaultMaxDepth
This file provides the interface for the instcombine pass implementation.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
static OverflowResult computeOverflowForSignedAdd(const WithCache< const Value * > &LHS, const WithCache< const Value * > &RHS, const AddOperator *Add, const SimplifyQuery &SQ)
Value * RHS
Value * LHS
BinaryOperator * Mul
support::ulittle16_t & Lo
Definition: aarch32.cpp:205
support::ulittle16_t & Hi
Definition: aarch32.cpp:204
static constexpr uint32_t Opcode
Definition: aarch32.h:200
static const uint32_t IV[8]
Definition: blake3_impl.h:78
Class for arbitrary precision integers.
Definition: APInt.h:76
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
Definition: Instructions.h:58
This class represents any memset intrinsic.
A cache of @llvm.assume calls within a function.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
This class represents a no-op cast from one type to another.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1227
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:451
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:738
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:748
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:165
This instruction extracts a single (scalar) element from a VectorType value.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This class represents a cast from floating point to signed integer.
This class represents a cast from floating point to unsigned integer.
This class represents a truncation of floating point types.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
An instruction for ordering other memory operations.
Definition: Instructions.h:436
This class represents a freeze function that returns random concrete value if an operand is either a ...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:940
This instruction compares its operands according to the predicate given to the constructor.
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
bool fmulByZeroIsZero(Value *MulVal, FastMathFlags FMF, const Instruction *CtxI) const
Check if fmul MulVal, +0.0 will yield +0.0 (or signed zero is ignorable).
virtual ~InstCombinerImpl()=default
KnownFPClass computeKnownFPClass(Value *Val, FastMathFlags FMF, FPClassTest Interested=fcAllFlags, const Instruction *CtxI=nullptr, unsigned Depth=0) const
Instruction * foldVectorSelect(SelectInst &Sel)
Instruction * foldSelectValueEquivalence(SelectInst &SI, ICmpInst &ICI)
Instruction * foldSPFofSPF(Instruction *Inner, SelectPatternFlavor SPF1, Value *A, Value *B, Instruction &Outer, SelectPatternFlavor SPF2, Value *C)
Constant * getLosslessUnsignedTrunc(Constant *C, Type *TruncTy)
bool replaceInInstruction(Value *V, Value *Old, Value *New, unsigned Depth=0)
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * foldSelectInstWithICmp(SelectInst &SI, ICmpInst *ICI)
Constant * getLosslessTrunc(Constant *C, Type *TruncTy, unsigned ExtOp)
Instruction * visitInstruction(Instruction &I)
Specify what to return for unhandled instructions.
KnownFPClass computeKnownFPClass(Value *Val, FPClassTest Interested=fcAllFlags, const Instruction *CtxI=nullptr, unsigned Depth=0) const
Constant * getLosslessSignedTrunc(Constant *C, Type *TruncTy)
InstCombinerImpl(InstructionWorklist &Worklist, BuilderTy &Builder, bool MinimizeSize, AAResults *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI)
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Instruction * visitSelectInst(SelectInst &SI)
Instruction * foldSelectOfBools(SelectInst &SI)
Instruction * foldSelectExtConst(SelectInst &Sel)
The core instruction combiner logic.
Definition: InstCombiner.h:46
Base class for instruction visitors.
Definition: InstVisitor.h:78
InstructionWorklist - This is the worklist management logic for InstCombine and other simplification ...
This class represents a cast from an integer to a pointer.
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:177
This class represents min/max intrinsics.
static Value * Negate(bool LHSIsZero, bool IsNSW, Value *Root, InstCombinerImpl &IC)
Attempt to negate Root.
This is a utility class that provides an abstraction for the common functionality between Instruction...
Definition: Operator.h:31
The optimization diagnostic interface.
Analysis providing profile information.
This class represents a cast from a pointer to an integer.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
Multiway switch.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
This function has undefined behavior.
This represents the llvm.va_end intrinsic.
LLVM Value Representation.
Definition: Value.h:74
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
This class represents zero extension of integer types.
self_iterator getIterator()
Definition: ilist_node.h:109
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
OverflowResult
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1394
OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, const SimplifyQuery &SQ)
OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS, const SimplifyQuery &SQ)
SelectPatternFlavor
Specific patterns of select instructions we can match.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS, const SimplifyQuery &SQ)
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
OverflowResult computeOverflowForUnsignedSub(const Value *LHS, const Value *RHS, const SimplifyQuery &SQ)
KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, const DataLayout &DL, FPClassTest InterestedClasses=fcAllFlags, unsigned Depth=0, const TargetLibraryInfo *TLI=nullptr, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.