LLVM 18.0.0git
ExpandVectorPredication.cpp
Go to the documentation of this file.
1//===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR expansion for vector predication intrinsics, allowing
10// targets to enable vector predication until just before codegen.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/Passes.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/Intrinsics.h"
28#include "llvm/Pass.h"
31#include "llvm/Support/Debug.h"
32#include <optional>
33
34using namespace llvm;
35
38
39// Keep this in sync with TargetTransformInfo::VPLegalization.
40#define VPINTERNAL_VPLEGAL_CASES \
41 VPINTERNAL_CASE(Legal) \
42 VPINTERNAL_CASE(Discard) \
43 VPINTERNAL_CASE(Convert)
44
45#define VPINTERNAL_CASE(X) "|" #X
46
47// Override options.
49 "expandvp-override-evl-transform", cl::init(""), cl::Hidden,
50 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
51 ". If non-empty, ignore "
52 "TargetTransformInfo and "
53 "always use this transformation for the %evl parameter (Used in "
54 "testing)."));
55
57 "expandvp-override-mask-transform", cl::init(""), cl::Hidden,
58 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
59 ". If non-empty, Ignore "
60 "TargetTransformInfo and "
61 "always use this transformation for the %mask parameter (Used in "
62 "testing)."));
63
64#undef VPINTERNAL_CASE
65#define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
66
67static VPTransform parseOverrideOption(const std::string &TextOpt) {
69}
70
71#undef VPINTERNAL_VPLEGAL_CASES
72
73// Whether any override options are set.
75 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
76}
77
78#define DEBUG_TYPE "expandvp"
79
80STATISTIC(NumFoldedVL, "Number of folded vector length params");
81STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
82
83///// Helpers {
84
85/// \returns Whether the vector mask \p MaskVal has all lane bits set.
86static bool isAllTrueMask(Value *MaskVal) {
87 if (Value *SplattedVal = getSplatValue(MaskVal))
88 if (auto *ConstValue = dyn_cast<Constant>(SplattedVal))
89 return ConstValue->isAllOnesValue();
90
91 return false;
92}
93
94/// \returns A non-excepting divisor constant for this type.
95static Constant *getSafeDivisor(Type *DivTy) {
96 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
97 return ConstantInt::get(DivTy, 1u, false);
98}
99
100/// Transfer operation properties from \p OldVPI to \p NewVal.
101static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
102 auto *NewInst = dyn_cast<Instruction>(&NewVal);
103 if (!NewInst || !isa<FPMathOperator>(NewVal))
104 return;
105
106 auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI);
107 if (!OldFMOp)
108 return;
109
110 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
111}
112
113/// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
114/// OldVP gets erased.
115static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
116 transferDecorations(NewOp, OldOp);
117 OldOp.replaceAllUsesWith(&NewOp);
118 OldOp.eraseFromParent();
119}
120
122 // The result of VP reductions depends on the mask and evl.
123 if (isa<VPReductionIntrinsic>(VPI))
124 return false;
125 // Fallback to whether the intrinsic is speculatable.
126 std::optional<unsigned> OpcOpt = VPI.getFunctionalOpcode();
127 unsigned FunctionalOpc = OpcOpt.value_or((unsigned)Instruction::Call);
128 return isSafeToSpeculativelyExecuteWithOpcode(FunctionalOpc, &VPI);
129}
130
131//// } Helpers
132
133namespace {
134
135// Expansion pass state at function scope.
136struct CachingVPExpander {
137 Function &F;
139
140 /// \returns A (fixed length) vector with ascending integer indices
141 /// (<0, 1, ..., NumElems-1>).
142 /// \p Builder
143 /// Used for instruction creation.
144 /// \p LaneTy
145 /// Integer element type of the result vector.
146 /// \p NumElems
147 /// Number of vector elements.
148 Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy,
149 unsigned NumElems);
150
151 /// \returns A bitmask that is true where the lane position is less-than \p
152 /// EVLParam
153 ///
154 /// \p Builder
155 /// Used for instruction creation.
156 /// \p VLParam
157 /// The explicit vector length parameter to test against the lane
158 /// positions.
159 /// \p ElemCount
160 /// Static (potentially scalable) number of vector elements.
161 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
162 ElementCount ElemCount);
163
164 Value *foldEVLIntoMask(VPIntrinsic &VPI);
165
166 /// "Remove" the %evl parameter of \p PI by setting it to the static vector
167 /// length of the operation.
168 void discardEVLParameter(VPIntrinsic &PI);
169
170 /// Lower this VP binary operator to a unpredicated binary operator.
171 Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
172 VPIntrinsic &PI);
173
174 /// Lower this VP int call to a unpredicated int call.
175 Value *expandPredicationToIntCall(IRBuilder<> &Builder, VPIntrinsic &PI,
176 unsigned UnpredicatedIntrinsicID);
177
178 /// Lower this VP fp call to a unpredicated fp call.
179 Value *expandPredicationToFPCall(IRBuilder<> &Builder, VPIntrinsic &PI,
180 unsigned UnpredicatedIntrinsicID);
181
182 /// Lower this VP reduction to a call to an unpredicated reduction intrinsic.
183 Value *expandPredicationInReduction(IRBuilder<> &Builder,
185
186 /// Lower this VP cast operation to a non-VP intrinsic.
187 Value *expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
188 VPIntrinsic &VPI);
189
190 /// Lower this VP memory operation to a non-VP intrinsic.
191 Value *expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
192 VPIntrinsic &VPI);
193
194 /// Lower this VP comparison to a call to an unpredicated comparison.
195 Value *expandPredicationInComparison(IRBuilder<> &Builder,
196 VPCmpIntrinsic &PI);
197
198 /// Query TTI and expand the vector predication in \p P accordingly.
199 Value *expandPredication(VPIntrinsic &PI);
200
201 /// Determine how and whether the VPIntrinsic \p VPI shall be expanded. This
202 /// overrides TTI with the cl::opts listed at the top of this file.
203 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
204 bool UsingTTIOverrides;
205
206public:
207 CachingVPExpander(Function &F, const TargetTransformInfo &TTI)
208 : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
209
210 bool expandVectorPredication();
211};
212
213//// CachingVPExpander {
214
215Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy,
216 unsigned NumElems) {
217 // TODO add caching
219
220 for (unsigned Idx = 0; Idx < NumElems; ++Idx)
221 ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false));
222
223 return ConstantVector::get(ConstElems);
224}
225
226Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
227 Value *EVLParam,
228 ElementCount ElemCount) {
229 // TODO add caching
230 // Scalable vector %evl conversion.
231 if (ElemCount.isScalable()) {
232 auto *M = Builder.GetInsertBlock()->getModule();
233 Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
234 Function *ActiveMaskFunc = Intrinsic::getDeclaration(
235 M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()});
236 // `get_active_lane_mask` performs an implicit less-than comparison.
237 Value *ConstZero = Builder.getInt32(0);
238 return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam});
239 }
240
241 // Fixed vector %evl conversion.
242 Type *LaneTy = EVLParam->getType();
243 unsigned NumElems = ElemCount.getFixedValue();
244 Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);
245 Value *IdxVec = createStepVector(Builder, LaneTy, NumElems);
246 return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);
247}
248
249Value *
250CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
251 VPIntrinsic &VPI) {
253 "Implicitly dropping %evl in non-speculatable operator!");
254
255 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
257
258 Value *Op0 = VPI.getOperand(0);
259 Value *Op1 = VPI.getOperand(1);
260 Value *Mask = VPI.getMaskParam();
261
262 // Blend in safe operands.
263 if (Mask && !isAllTrueMask(Mask)) {
264 switch (OC) {
265 default:
266 // Can safely ignore the predicate.
267 break;
268
269 // Division operators need a safe divisor on masked-off lanes (1).
270 case Instruction::UDiv:
271 case Instruction::SDiv:
272 case Instruction::URem:
273 case Instruction::SRem:
274 // 2nd operand must not be zero.
275 Value *SafeDivisor = getSafeDivisor(VPI.getType());
276 Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);
277 }
278 }
279
280 Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());
281
282 replaceOperation(*NewBinOp, VPI);
283 return NewBinOp;
284}
285
286Value *CachingVPExpander::expandPredicationToIntCall(
287 IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
288 switch (UnpredicatedIntrinsicID) {
289 case Intrinsic::abs:
290 case Intrinsic::smax:
291 case Intrinsic::smin:
292 case Intrinsic::umax:
293 case Intrinsic::umin: {
294 Value *Op0 = VPI.getOperand(0);
295 Value *Op1 = VPI.getOperand(1);
297 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
298 Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());
299 replaceOperation(*NewOp, VPI);
300 return NewOp;
301 }
302 }
303 return nullptr;
304}
305
306Value *CachingVPExpander::expandPredicationToFPCall(
307 IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
309 "Implicitly dropping %evl in non-speculatable operator!");
310
311 switch (UnpredicatedIntrinsicID) {
312 case Intrinsic::fabs:
313 case Intrinsic::sqrt: {
314 Value *Op0 = VPI.getOperand(0);
316 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
317 Value *NewOp = Builder.CreateCall(Fn, {Op0}, VPI.getName());
318 replaceOperation(*NewOp, VPI);
319 return NewOp;
320 }
321 case Intrinsic::maxnum:
322 case Intrinsic::minnum: {
323 Value *Op0 = VPI.getOperand(0);
324 Value *Op1 = VPI.getOperand(1);
326 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
327 Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());
328 replaceOperation(*NewOp, VPI);
329 return NewOp;
330 }
331 case Intrinsic::experimental_constrained_fma:
332 case Intrinsic::experimental_constrained_fmuladd: {
333 Value *Op0 = VPI.getOperand(0);
334 Value *Op1 = VPI.getOperand(1);
335 Value *Op2 = VPI.getOperand(2);
337 VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
338 Value *NewOp =
339 Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName());
340 replaceOperation(*NewOp, VPI);
341 return NewOp;
342 }
343 }
344
345 return nullptr;
346}
347
348static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
349 Type *EltTy) {
350 bool Negative = false;
351 unsigned EltBits = EltTy->getScalarSizeInBits();
352 switch (VPI.getIntrinsicID()) {
353 default:
354 llvm_unreachable("Expecting a VP reduction intrinsic");
355 case Intrinsic::vp_reduce_add:
356 case Intrinsic::vp_reduce_or:
357 case Intrinsic::vp_reduce_xor:
358 case Intrinsic::vp_reduce_umax:
359 return Constant::getNullValue(EltTy);
360 case Intrinsic::vp_reduce_mul:
361 return ConstantInt::get(EltTy, 1, /*IsSigned*/ false);
362 case Intrinsic::vp_reduce_and:
363 case Intrinsic::vp_reduce_umin:
364 return ConstantInt::getAllOnesValue(EltTy);
365 case Intrinsic::vp_reduce_smin:
366 return ConstantInt::get(EltTy->getContext(),
367 APInt::getSignedMaxValue(EltBits));
368 case Intrinsic::vp_reduce_smax:
369 return ConstantInt::get(EltTy->getContext(),
370 APInt::getSignedMinValue(EltBits));
371 case Intrinsic::vp_reduce_fmax:
372 Negative = true;
373 [[fallthrough]];
374 case Intrinsic::vp_reduce_fmin: {
376 const fltSemantics &Semantics = EltTy->getFltSemantics();
377 return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative)
378 : !Flags.noInfs()
379 ? ConstantFP::getInfinity(EltTy, Negative)
380 : ConstantFP::get(EltTy,
381 APFloat::getLargest(Semantics, Negative));
382 }
383 case Intrinsic::vp_reduce_fadd:
384 return ConstantFP::getNegativeZero(EltTy);
385 case Intrinsic::vp_reduce_fmul:
386 return ConstantFP::get(EltTy, 1.0);
387 }
388}
389
390Value *
391CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
394 "Implicitly dropping %evl in non-speculatable operator!");
395
396 Value *Mask = VPI.getMaskParam();
397 Value *RedOp = VPI.getOperand(VPI.getVectorParamPos());
398
399 // Insert neutral element in masked-out positions
400 if (Mask && !isAllTrueMask(Mask)) {
401 auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType());
402 auto *NeutralVector = Builder.CreateVectorSplat(
403 cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt);
404 RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);
405 }
406
408 Value *Start = VPI.getOperand(VPI.getStartParamPos());
409
410 switch (VPI.getIntrinsicID()) {
411 default:
412 llvm_unreachable("Impossible reduction kind");
413 case Intrinsic::vp_reduce_add:
414 Reduction = Builder.CreateAddReduce(RedOp);
415 Reduction = Builder.CreateAdd(Reduction, Start);
416 break;
417 case Intrinsic::vp_reduce_mul:
418 Reduction = Builder.CreateMulReduce(RedOp);
419 Reduction = Builder.CreateMul(Reduction, Start);
420 break;
421 case Intrinsic::vp_reduce_and:
422 Reduction = Builder.CreateAndReduce(RedOp);
423 Reduction = Builder.CreateAnd(Reduction, Start);
424 break;
425 case Intrinsic::vp_reduce_or:
426 Reduction = Builder.CreateOrReduce(RedOp);
427 Reduction = Builder.CreateOr(Reduction, Start);
428 break;
429 case Intrinsic::vp_reduce_xor:
430 Reduction = Builder.CreateXorReduce(RedOp);
431 Reduction = Builder.CreateXor(Reduction, Start);
432 break;
433 case Intrinsic::vp_reduce_smax:
434 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true);
435 Reduction =
436 Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start);
437 break;
438 case Intrinsic::vp_reduce_smin:
439 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true);
440 Reduction =
441 Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start);
442 break;
443 case Intrinsic::vp_reduce_umax:
444 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false);
445 Reduction =
446 Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start);
447 break;
448 case Intrinsic::vp_reduce_umin:
449 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false);
450 Reduction =
451 Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start);
452 break;
453 case Intrinsic::vp_reduce_fmax:
454 Reduction = Builder.CreateFPMaxReduce(RedOp);
455 transferDecorations(*Reduction, VPI);
456 Reduction =
457 Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start);
458 break;
459 case Intrinsic::vp_reduce_fmin:
460 Reduction = Builder.CreateFPMinReduce(RedOp);
461 transferDecorations(*Reduction, VPI);
462 Reduction =
463 Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);
464 break;
465 case Intrinsic::vp_reduce_fadd:
466 Reduction = Builder.CreateFAddReduce(Start, RedOp);
467 break;
468 case Intrinsic::vp_reduce_fmul:
469 Reduction = Builder.CreateFMulReduce(Start, RedOp);
470 break;
471 }
472
473 replaceOperation(*Reduction, VPI);
474 return Reduction;
475}
476
477Value *CachingVPExpander::expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
478 VPIntrinsic &VPI) {
479 Value *CastOp = nullptr;
480 switch (VPI.getIntrinsicID()) {
481 default:
482 llvm_unreachable("Not a VP memory intrinsic");
483 case Intrinsic::vp_sext:
484 CastOp =
485 Builder.CreateSExt(VPI.getOperand(0), VPI.getType(), VPI.getName());
486 break;
487 case Intrinsic::vp_zext:
488 CastOp =
489 Builder.CreateZExt(VPI.getOperand(0), VPI.getType(), VPI.getName());
490 break;
491 case Intrinsic::vp_trunc:
492 CastOp =
493 Builder.CreateTrunc(VPI.getOperand(0), VPI.getType(), VPI.getName());
494 break;
495 case Intrinsic::vp_inttoptr:
496 CastOp =
497 Builder.CreateIntToPtr(VPI.getOperand(0), VPI.getType(), VPI.getName());
498 break;
499 case Intrinsic::vp_ptrtoint:
500 CastOp =
501 Builder.CreatePtrToInt(VPI.getOperand(0), VPI.getType(), VPI.getName());
502 break;
503 case Intrinsic::vp_fptosi:
504 CastOp =
505 Builder.CreateFPToSI(VPI.getOperand(0), VPI.getType(), VPI.getName());
506 break;
507
508 case Intrinsic::vp_fptoui:
509 CastOp =
510 Builder.CreateFPToUI(VPI.getOperand(0), VPI.getType(), VPI.getName());
511 break;
512 case Intrinsic::vp_sitofp:
513 CastOp =
514 Builder.CreateSIToFP(VPI.getOperand(0), VPI.getType(), VPI.getName());
515 break;
516 case Intrinsic::vp_uitofp:
517 CastOp =
518 Builder.CreateUIToFP(VPI.getOperand(0), VPI.getType(), VPI.getName());
519 break;
520 case Intrinsic::vp_fptrunc:
521 CastOp =
522 Builder.CreateFPTrunc(VPI.getOperand(0), VPI.getType(), VPI.getName());
523 break;
524 case Intrinsic::vp_fpext:
525 CastOp =
526 Builder.CreateFPExt(VPI.getOperand(0), VPI.getType(), VPI.getName());
527 break;
528 }
529 replaceOperation(*CastOp, VPI);
530 return CastOp;
531}
532
533Value *
534CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
535 VPIntrinsic &VPI) {
537
538 const auto &DL = F.getParent()->getDataLayout();
539
540 Value *MaskParam = VPI.getMaskParam();
541 Value *PtrParam = VPI.getMemoryPointerParam();
542 Value *DataParam = VPI.getMemoryDataParam();
543 bool IsUnmasked = isAllTrueMask(MaskParam);
544
545 MaybeAlign AlignOpt = VPI.getPointerAlignment();
546
547 Value *NewMemoryInst = nullptr;
548 switch (VPI.getIntrinsicID()) {
549 default:
550 llvm_unreachable("Not a VP memory intrinsic");
551 case Intrinsic::vp_store:
552 if (IsUnmasked) {
553 StoreInst *NewStore =
554 Builder.CreateStore(DataParam, PtrParam, /*IsVolatile*/ false);
555 if (AlignOpt.has_value())
556 NewStore->setAlignment(*AlignOpt);
557 NewMemoryInst = NewStore;
558 } else
559 NewMemoryInst = Builder.CreateMaskedStore(
560 DataParam, PtrParam, AlignOpt.valueOrOne(), MaskParam);
561
562 break;
563 case Intrinsic::vp_load:
564 if (IsUnmasked) {
565 LoadInst *NewLoad =
566 Builder.CreateLoad(VPI.getType(), PtrParam, /*IsVolatile*/ false);
567 if (AlignOpt.has_value())
568 NewLoad->setAlignment(*AlignOpt);
569 NewMemoryInst = NewLoad;
570 } else
571 NewMemoryInst = Builder.CreateMaskedLoad(
572 VPI.getType(), PtrParam, AlignOpt.valueOrOne(), MaskParam);
573
574 break;
575 case Intrinsic::vp_scatter: {
576 auto *ElementType =
577 cast<VectorType>(DataParam->getType())->getElementType();
578 NewMemoryInst = Builder.CreateMaskedScatter(
579 DataParam, PtrParam,
580 AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam);
581 break;
582 }
583 case Intrinsic::vp_gather: {
584 auto *ElementType = cast<VectorType>(VPI.getType())->getElementType();
585 NewMemoryInst = Builder.CreateMaskedGather(
586 VPI.getType(), PtrParam,
587 AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam, nullptr,
588 VPI.getName());
589 break;
590 }
591 }
592
593 assert(NewMemoryInst);
594 replaceOperation(*NewMemoryInst, VPI);
595 return NewMemoryInst;
596}
597
598Value *CachingVPExpander::expandPredicationInComparison(IRBuilder<> &Builder,
599 VPCmpIntrinsic &VPI) {
601 "Implicitly dropping %evl in non-speculatable operator!");
602
603 assert(*VPI.getFunctionalOpcode() == Instruction::ICmp ||
604 *VPI.getFunctionalOpcode() == Instruction::FCmp);
605
606 Value *Op0 = VPI.getOperand(0);
607 Value *Op1 = VPI.getOperand(1);
608 auto Pred = VPI.getPredicate();
609
610 auto *NewCmp = Builder.CreateCmp(Pred, Op0, Op1);
611
612 replaceOperation(*NewCmp, VPI);
613 return NewCmp;
614}
615
616void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
617 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
618
620 return;
621
622 Value *EVLParam = VPI.getVectorLengthParam();
623 if (!EVLParam)
624 return;
625
626 ElementCount StaticElemCount = VPI.getStaticVectorLength();
627 Value *MaxEVL = nullptr;
629 if (StaticElemCount.isScalable()) {
630 // TODO add caching
631 auto *M = VPI.getModule();
632 Function *VScaleFunc =
633 Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty);
635 Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
636 Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");
637 MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
638 /*NUW*/ true, /*NSW*/ false);
639 } else {
640 MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
641 }
642 VPI.setVectorLengthParam(MaxEVL);
643}
644
645Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
646 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
647
648 IRBuilder<> Builder(&VPI);
649
650 // Ineffective %evl parameter and so nothing to do here.
652 return &VPI;
653
654 // Only VP intrinsics can have an %evl parameter.
655 Value *OldMaskParam = VPI.getMaskParam();
656 Value *OldEVLParam = VPI.getVectorLengthParam();
657 assert(OldMaskParam && "no mask param to fold the vl param into");
658 assert(OldEVLParam && "no EVL param to fold away");
659
660 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
661 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
662
663 // Convert the %evl predication into vector mask predication.
664 ElementCount ElemCount = VPI.getStaticVectorLength();
665 Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
666 Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);
667 VPI.setMaskParam(NewMaskParam);
668
669 // Drop the %evl parameter.
670 discardEVLParameter(VPI);
672 "transformation did not render the evl param ineffective!");
673
674 // Reassess the modified instruction.
675 return &VPI;
676}
677
678Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
679 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
680
681 IRBuilder<> Builder(&VPI);
682
683 // Try lowering to a LLVM instruction first.
684 auto OC = VPI.getFunctionalOpcode();
685
686 if (OC && Instruction::isBinaryOp(*OC))
687 return expandPredicationInBinaryOperator(Builder, VPI);
688
689 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI))
690 return expandPredicationInReduction(Builder, *VPRI);
691
692 if (auto *VPCmp = dyn_cast<VPCmpIntrinsic>(&VPI))
693 return expandPredicationInComparison(Builder, *VPCmp);
694
696 return expandPredicationToCastIntrinsic(Builder, VPI);
697 }
698
699 switch (VPI.getIntrinsicID()) {
700 default:
701 break;
702 case Intrinsic::vp_fneg: {
703 Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0), VPI.getName());
704 replaceOperation(*NewNegOp, VPI);
705 return NewNegOp;
706 }
707 case Intrinsic::vp_abs:
708 return expandPredicationToIntCall(Builder, VPI, Intrinsic::abs);
709 case Intrinsic::vp_smax:
710 return expandPredicationToIntCall(Builder, VPI, Intrinsic::smax);
711 case Intrinsic::vp_smin:
712 return expandPredicationToIntCall(Builder, VPI, Intrinsic::smin);
713 case Intrinsic::vp_umax:
714 return expandPredicationToIntCall(Builder, VPI, Intrinsic::umax);
715 case Intrinsic::vp_umin:
716 return expandPredicationToIntCall(Builder, VPI, Intrinsic::umin);
717 case Intrinsic::vp_fabs:
718 return expandPredicationToFPCall(Builder, VPI, Intrinsic::fabs);
719 case Intrinsic::vp_sqrt:
720 return expandPredicationToFPCall(Builder, VPI, Intrinsic::sqrt);
721 case Intrinsic::vp_maxnum:
722 return expandPredicationToFPCall(Builder, VPI, Intrinsic::maxnum);
723 case Intrinsic::vp_minnum:
724 return expandPredicationToFPCall(Builder, VPI, Intrinsic::minnum);
725 case Intrinsic::vp_load:
726 case Intrinsic::vp_store:
727 case Intrinsic::vp_gather:
728 case Intrinsic::vp_scatter:
729 return expandPredicationInMemoryIntrinsic(Builder, VPI);
730 }
731
732 if (auto CID = VPI.getConstrainedIntrinsicID())
733 if (Value *Call = expandPredicationToFPCall(Builder, VPI, *CID))
734 return Call;
735
736 return &VPI;
737}
738
739//// } CachingVPExpander
740
741struct TransformJob {
742 VPIntrinsic *PI;
744 TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat)
745 : PI(PI), Strategy(InitStrat) {}
746
747 bool isDone() const { return Strategy.shouldDoNothing(); }
748};
749
750void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
751 // Operations with speculatable lanes do not strictly need predication.
752 if (maySpeculateLanes(VPI)) {
753 // Converting a speculatable VP intrinsic means dropping %mask and %evl.
754 // No need to expand %evl into the %mask only to ignore that code.
755 if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
757 return;
758 }
759
760 // We have to preserve the predicating effect of %evl for this
761 // non-speculatable VP intrinsic.
762 // 1) Never discard %evl.
763 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
764 // %evl gets folded into %mask.
765 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
766 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
768 }
769}
770
772CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
773 auto VPStrat = TTI.getVPLegalizationStrategy(VPI);
774 if (LLVM_LIKELY(!UsingTTIOverrides)) {
775 // No overrides - we are in production.
776 return VPStrat;
777 }
778
779 // Overrides set - we are in testing, the following does not need to be
780 // efficient.
782 VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride);
783 return VPStrat;
784}
785
786/// Expand llvm.vp.* intrinsics as requested by \p TTI.
787bool CachingVPExpander::expandVectorPredication() {
789
790 // Collect all VPIntrinsics that need expansion and determine their expansion
791 // strategy.
792 for (auto &I : instructions(F)) {
793 auto *VPI = dyn_cast<VPIntrinsic>(&I);
794 if (!VPI)
795 continue;
796 auto VPStrat = getVPLegalizationStrategy(*VPI);
797 sanitizeStrategy(*VPI, VPStrat);
798 if (!VPStrat.shouldDoNothing())
799 Worklist.emplace_back(VPI, VPStrat);
800 }
801 if (Worklist.empty())
802 return false;
803
804 // Transform all VPIntrinsics on the worklist.
805 LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size()
806 << " instructions ::::\n");
807 for (TransformJob Job : Worklist) {
808 // Transform the EVL parameter.
809 switch (Job.Strategy.EVLParamStrategy) {
811 break;
813 discardEVLParameter(*Job.PI);
814 break;
816 if (foldEVLIntoMask(*Job.PI))
817 ++NumFoldedVL;
818 break;
819 }
820 Job.Strategy.EVLParamStrategy = VPLegalization::Legal;
821
822 // Replace with a non-predicated operation.
823 switch (Job.Strategy.OpStrategy) {
825 break;
827 llvm_unreachable("Invalid strategy for operators.");
829 expandPredication(*Job.PI);
830 ++NumLoweredVPOps;
831 break;
832 }
833 Job.Strategy.OpStrategy = VPLegalization::Legal;
834
835 assert(Job.isDone() && "incomplete transformation");
836 }
837
838 return true;
839}
840class ExpandVectorPredication : public FunctionPass {
841public:
842 static char ID;
843 ExpandVectorPredication() : FunctionPass(ID) {
845 }
846
847 bool runOnFunction(Function &F) override {
848 const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
849 CachingVPExpander VPExpander(F, *TTI);
850 return VPExpander.expandVectorPredication();
851 }
852
853 void getAnalysisUsage(AnalysisUsage &AU) const override {
855 AU.setPreservesCFG();
856 }
857};
858} // namespace
859
860char ExpandVectorPredication::ID;
861INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp",
862 "Expand vector predication intrinsics", false, false)
865INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp",
867
869 return new ExpandVectorPredication();
870}
871
874 const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
875 CachingVPExpander VPExpander(F, TTI);
876 if (!VPExpander.expandVectorPredication())
877 return PreservedAnalyses::all();
880 return PA;
881}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
assume Assume Builder
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:221
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static VPTransform parseOverrideOption(const std::string &TextOpt)
static cl::opt< std::string > MaskTransformOverride("expandvp-override-mask-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, Ignore " "TargetTransformInfo and " "always use this transformation for the %mask parameter (Used in " "testing)."))
static cl::opt< std::string > EVLTransformOverride("expandvp-override-evl-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, ignore " "TargetTransformInfo and " "always use this transformation for the %evl parameter (Used in " "testing)."))
static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp)
Transfer all properties from OldOp to NewOp and replace all uses.
static bool isAllTrueMask(Value *MaskVal)
static void transferDecorations(Value &NewVal, VPIntrinsic &VPI)
Transfer operation properties from OldVPI to NewVal.
static bool anyExpandVPOverridesSet()
static bool maySpeculateLanes(VPIntrinsic &VPI)
Expand vector predication intrinsics
static Constant * getSafeDivisor(Type *DivTy)
#define VPINTERNAL_VPLEGAL_CASES
Select target instructions out of generic instructions
loop predication
loop Loop Strength Reduction
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
IntegerType * Int32Ty
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This pass exposes codegen information to IR-level passes.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:269
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:113
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:736
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:260
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:927
static Constant * getNegativeZero(Type *Ty)
Definition: Constants.h:291
static Constant * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)
Definition: Constants.cpp:979
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1342
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:356
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:314
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2625
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:71
bool isBinaryOp() const
Definition: Instruction.h:200
const BasicBlock * getParent() const
Definition: Instruction.h:90
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:83
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
An instruction for reading from memory.
Definition: Instructions.h:177
void setAlignment(Align Align)
Definition: Instructions.h:224
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:941
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
void setAlignment(Align Align)
Definition: Instructions.h:349
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:44
Analysis pass providing the TargetTransformInfo.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
const fltSemantics & getFltSemantics() const
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:234
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static IntegerType * getInt32Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Definition: User.h:169
static bool isVPCast(Intrinsic::ID ID)
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
bool canIgnoreVectorLengthParam() const
void setMaskParam(Value *)
Value * getVectorLengthParam() const
void setVectorLengthParam(Value *)
Value * getMemoryDataParam() const
Value * getMemoryPointerParam() const
std::optional< unsigned > getConstrainedIntrinsicID() const
MaybeAlign getPointerAlignment() const
Value * getMaskParam() const
ElementCount getStaticVectorLength() const
std::optional< unsigned > getFunctionalOpcode() const
This represents vector predication reduction intrinsics.
unsigned getStartParamPos() const
unsigned getVectorParamPos() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:535
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1069
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:182
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:166
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:163
self_iterator getIterator()
Definition: ilist_node.h:82
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1422
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
FunctionPass * createExpandVectorPredicationPass()
This pass expands the vector predication intrinsics into unpredicated instructions with selects or ju...
void initializeExpandVectorPredicationPass(PassRegistry &)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isSafeToSpeculativelyExecuteWithOpcode(unsigned Opcode, const Instruction *Inst, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
This returns the same result as isSafeToSpeculativelyExecute if Opcode is the actual opcode of Inst.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141