LLVM 20.0.0git
X86FixupVectorConstants.cpp
Go to the documentation of this file.
1//===-- X86FixupVectorConstants.cpp - optimize constant generation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file examines all full size vector constant pool loads and attempts to
10// replace them with smaller constant pool entries, including:
11// * Converting AVX512 memory-fold instructions to their broadcast-fold form.
12// * Using vzload scalar loads.
13// * Broadcasting of full width loads.
14// * Sign/Zero extension of full width loads.
15//
16//===----------------------------------------------------------------------===//
17
18#include "X86.h"
19#include "X86InstrFoldTables.h"
20#include "X86InstrInfo.h"
21#include "X86Subtarget.h"
22#include "llvm/ADT/Statistic.h"
24
25using namespace llvm;
26
27#define DEBUG_TYPE "x86-fixup-vector-constants"
28
29STATISTIC(NumInstChanges, "Number of instructions changes");
30
31namespace {
32class X86FixupVectorConstantsPass : public MachineFunctionPass {
33public:
34 static char ID;
35
36 X86FixupVectorConstantsPass() : MachineFunctionPass(ID) {}
37
38 StringRef getPassName() const override {
39 return "X86 Fixup Vector Constants";
40 }
41
42 bool runOnMachineFunction(MachineFunction &MF) override;
43 bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
45
46 // This pass runs after regalloc and doesn't support VReg operands.
49 MachineFunctionProperties::Property::NoVRegs);
50 }
51
52private:
53 const X86InstrInfo *TII = nullptr;
54 const X86Subtarget *ST = nullptr;
55 const MCSchedModel *SM = nullptr;
56};
57} // end anonymous namespace
58
59char X86FixupVectorConstantsPass::ID = 0;
60
61INITIALIZE_PASS(X86FixupVectorConstantsPass, DEBUG_TYPE, DEBUG_TYPE, false, false)
62
64 return new X86FixupVectorConstantsPass();
65}
66
67/// Normally, we only allow poison in vector splats. However, as this is part
68/// of the backend, and working with the DAG representation, which currently
69/// only natively represents undef values, we need to accept undefs here.
71 Constant *Res = nullptr;
72 for (Value *Op : C->operands()) {
73 Constant *OpC = cast<Constant>(Op);
74 if (isa<UndefValue>(OpC))
75 continue;
76 if (!Res)
77 Res = OpC;
78 else if (Res != OpC)
79 return nullptr;
80 }
81 return Res;
82}
83
84// Attempt to extract the full width of bits data from the constant.
85static std::optional<APInt> extractConstantBits(const Constant *C) {
86 unsigned NumBits = C->getType()->getPrimitiveSizeInBits();
87
88 if (isa<UndefValue>(C))
89 return APInt::getZero(NumBits);
90
91 if (auto *CInt = dyn_cast<ConstantInt>(C))
92 return CInt->getValue();
93
94 if (auto *CFP = dyn_cast<ConstantFP>(C))
95 return CFP->getValue().bitcastToAPInt();
96
97 if (auto *CV = dyn_cast<ConstantVector>(C)) {
98 if (auto *CVSplat = getSplatValueAllowUndef(CV)) {
99 if (std::optional<APInt> Bits = extractConstantBits(CVSplat)) {
100 assert((NumBits % Bits->getBitWidth()) == 0 && "Illegal splat");
101 return APInt::getSplat(NumBits, *Bits);
102 }
103 }
104
105 APInt Bits = APInt::getZero(NumBits);
106 for (unsigned I = 0, E = CV->getNumOperands(); I != E; ++I) {
107 Constant *Elt = CV->getOperand(I);
108 std::optional<APInt> SubBits = extractConstantBits(Elt);
109 if (!SubBits)
110 return std::nullopt;
111 assert(NumBits == (E * SubBits->getBitWidth()) &&
112 "Illegal vector element size");
113 Bits.insertBits(*SubBits, I * SubBits->getBitWidth());
114 }
115 return Bits;
116 }
117
118 if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
119 bool IsInteger = CDS->getElementType()->isIntegerTy();
120 bool IsFloat = CDS->getElementType()->isHalfTy() ||
121 CDS->getElementType()->isBFloatTy() ||
122 CDS->getElementType()->isFloatTy() ||
123 CDS->getElementType()->isDoubleTy();
124 if (IsInteger || IsFloat) {
125 APInt Bits = APInt::getZero(NumBits);
126 unsigned EltBits = CDS->getElementType()->getPrimitiveSizeInBits();
127 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
128 if (IsInteger)
129 Bits.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
130 else
131 Bits.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
132 I * EltBits);
133 }
134 return Bits;
135 }
136 }
137
138 return std::nullopt;
139}
140
141static std::optional<APInt> extractConstantBits(const Constant *C,
142 unsigned NumBits) {
143 if (std::optional<APInt> Bits = extractConstantBits(C))
144 return Bits->zextOrTrunc(NumBits);
145 return std::nullopt;
146}
147
148// Attempt to compute the splat width of bits data by normalizing the splat to
149// remove undefs.
150static std::optional<APInt> getSplatableConstant(const Constant *C,
151 unsigned SplatBitWidth) {
152 const Type *Ty = C->getType();
153 assert((Ty->getPrimitiveSizeInBits() % SplatBitWidth) == 0 &&
154 "Illegal splat width");
155
156 if (std::optional<APInt> Bits = extractConstantBits(C))
157 if (Bits->isSplat(SplatBitWidth))
158 return Bits->trunc(SplatBitWidth);
159
160 // Detect general splats with undefs.
161 // TODO: Do we need to handle NumEltsBits > SplatBitWidth splitting?
162 if (auto *CV = dyn_cast<ConstantVector>(C)) {
163 unsigned NumOps = CV->getNumOperands();
164 unsigned NumEltsBits = Ty->getScalarSizeInBits();
165 unsigned NumScaleOps = SplatBitWidth / NumEltsBits;
166 if ((SplatBitWidth % NumEltsBits) == 0) {
167 // Collect the elements and ensure that within the repeated splat sequence
168 // they either match or are undef.
169 SmallVector<Constant *, 16> Sequence(NumScaleOps, nullptr);
170 for (unsigned Idx = 0; Idx != NumOps; ++Idx) {
171 if (Constant *Elt = CV->getAggregateElement(Idx)) {
172 if (isa<UndefValue>(Elt))
173 continue;
174 unsigned SplatIdx = Idx % NumScaleOps;
175 if (!Sequence[SplatIdx] || Sequence[SplatIdx] == Elt) {
176 Sequence[SplatIdx] = Elt;
177 continue;
178 }
179 }
180 return std::nullopt;
181 }
182 // Extract the constant bits forming the splat and insert into the bits
183 // data, leave undef as zero.
184 APInt SplatBits = APInt::getZero(SplatBitWidth);
185 for (unsigned I = 0; I != NumScaleOps; ++I) {
186 if (!Sequence[I])
187 continue;
188 if (std::optional<APInt> Bits = extractConstantBits(Sequence[I])) {
189 SplatBits.insertBits(*Bits, I * Bits->getBitWidth());
190 continue;
191 }
192 return std::nullopt;
193 }
194 return SplatBits;
195 }
196 }
197
198 return std::nullopt;
199}
200
201// Split raw bits into a constant vector of elements of a specific bit width.
202// NOTE: We don't always bother converting to scalars if the vector length is 1.
204 const APInt &Bits, unsigned NumSclBits) {
205 unsigned BitWidth = Bits.getBitWidth();
206
207 if (NumSclBits == 8) {
208 SmallVector<uint8_t> RawBits;
209 for (unsigned I = 0; I != BitWidth; I += 8)
210 RawBits.push_back(Bits.extractBits(8, I).getZExtValue());
211 return ConstantDataVector::get(Ctx, RawBits);
212 }
213
214 if (NumSclBits == 16) {
215 SmallVector<uint16_t> RawBits;
216 for (unsigned I = 0; I != BitWidth; I += 16)
217 RawBits.push_back(Bits.extractBits(16, I).getZExtValue());
218 if (SclTy->is16bitFPTy())
219 return ConstantDataVector::getFP(SclTy, RawBits);
220 return ConstantDataVector::get(Ctx, RawBits);
221 }
222
223 if (NumSclBits == 32) {
224 SmallVector<uint32_t> RawBits;
225 for (unsigned I = 0; I != BitWidth; I += 32)
226 RawBits.push_back(Bits.extractBits(32, I).getZExtValue());
227 if (SclTy->isFloatTy())
228 return ConstantDataVector::getFP(SclTy, RawBits);
229 return ConstantDataVector::get(Ctx, RawBits);
230 }
231
232 assert(NumSclBits == 64 && "Unhandled vector element width");
233
234 SmallVector<uint64_t> RawBits;
235 for (unsigned I = 0; I != BitWidth; I += 64)
236 RawBits.push_back(Bits.extractBits(64, I).getZExtValue());
237 if (SclTy->isDoubleTy())
238 return ConstantDataVector::getFP(SclTy, RawBits);
239 return ConstantDataVector::get(Ctx, RawBits);
240}
241
242// Attempt to rebuild a normalized splat vector constant of the requested splat
243// width, built up of potentially smaller scalar values.
244static Constant *rebuildSplatCst(const Constant *C, unsigned /*NumBits*/,
245 unsigned /*NumElts*/, unsigned SplatBitWidth) {
246 // TODO: Truncate to NumBits once ConvertToBroadcastAVX512 support this.
247 std::optional<APInt> Splat = getSplatableConstant(C, SplatBitWidth);
248 if (!Splat)
249 return nullptr;
250
251 // Determine scalar size to use for the constant splat vector, clamping as we
252 // might have found a splat smaller than the original constant data.
253 Type *SclTy = C->getType()->getScalarType();
254 unsigned NumSclBits = SclTy->getPrimitiveSizeInBits();
255 NumSclBits = std::min<unsigned>(NumSclBits, SplatBitWidth);
256
257 // Fallback to i64 / double.
258 NumSclBits = (NumSclBits == 8 || NumSclBits == 16 || NumSclBits == 32)
259 ? NumSclBits
260 : 64;
261
262 // Extract per-element bits.
263 return rebuildConstant(C->getContext(), SclTy, *Splat, NumSclBits);
264}
265
266static Constant *rebuildZeroUpperCst(const Constant *C, unsigned NumBits,
267 unsigned /*NumElts*/,
268 unsigned ScalarBitWidth) {
269 Type *SclTy = C->getType()->getScalarType();
270 unsigned NumSclBits = SclTy->getPrimitiveSizeInBits();
271 LLVMContext &Ctx = C->getContext();
272
273 if (NumBits > ScalarBitWidth) {
274 // Determine if the upper bits are all zero.
275 if (std::optional<APInt> Bits = extractConstantBits(C, NumBits)) {
276 if (Bits->countLeadingZeros() >= (NumBits - ScalarBitWidth)) {
277 // If the original constant was made of smaller elements, try to retain
278 // those types.
279 if (ScalarBitWidth > NumSclBits && (ScalarBitWidth % NumSclBits) == 0)
280 return rebuildConstant(Ctx, SclTy, *Bits, NumSclBits);
281
282 // Fallback to raw integer bits.
283 APInt RawBits = Bits->zextOrTrunc(ScalarBitWidth);
284 return ConstantInt::get(Ctx, RawBits);
285 }
286 }
287 }
288
289 return nullptr;
290}
291
292static Constant *rebuildExtCst(const Constant *C, bool IsSExt,
293 unsigned NumBits, unsigned NumElts,
294 unsigned SrcEltBitWidth) {
295 unsigned DstEltBitWidth = NumBits / NumElts;
296 assert((NumBits % NumElts) == 0 && (NumBits % SrcEltBitWidth) == 0 &&
297 (DstEltBitWidth % SrcEltBitWidth) == 0 &&
298 (DstEltBitWidth > SrcEltBitWidth) && "Illegal extension width");
299
300 if (std::optional<APInt> Bits = extractConstantBits(C, NumBits)) {
301 assert((Bits->getBitWidth() / DstEltBitWidth) == NumElts &&
302 (Bits->getBitWidth() % DstEltBitWidth) == 0 &&
303 "Unexpected constant extension");
304
305 // Ensure every vector element can be represented by the src bitwidth.
306 APInt TruncBits = APInt::getZero(NumElts * SrcEltBitWidth);
307 for (unsigned I = 0; I != NumElts; ++I) {
308 APInt Elt = Bits->extractBits(DstEltBitWidth, I * DstEltBitWidth);
309 if ((IsSExt && Elt.getSignificantBits() > SrcEltBitWidth) ||
310 (!IsSExt && Elt.getActiveBits() > SrcEltBitWidth))
311 return nullptr;
312 TruncBits.insertBits(Elt.trunc(SrcEltBitWidth), I * SrcEltBitWidth);
313 }
314
315 Type *Ty = C->getType();
316 return rebuildConstant(Ty->getContext(), Ty->getScalarType(), TruncBits,
317 SrcEltBitWidth);
318 }
319
320 return nullptr;
321}
322static Constant *rebuildSExtCst(const Constant *C, unsigned NumBits,
323 unsigned NumElts, unsigned SrcEltBitWidth) {
324 return rebuildExtCst(C, true, NumBits, NumElts, SrcEltBitWidth);
325}
326static Constant *rebuildZExtCst(const Constant *C, unsigned NumBits,
327 unsigned NumElts, unsigned SrcEltBitWidth) {
328 return rebuildExtCst(C, false, NumBits, NumElts, SrcEltBitWidth);
329}
330
331bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
333 MachineInstr &MI) {
334 unsigned Opc = MI.getOpcode();
335 MachineConstantPool *CP = MI.getParent()->getParent()->getConstantPool();
336 bool HasSSE41 = ST->hasSSE41();
337 bool HasAVX2 = ST->hasAVX2();
338 bool HasDQI = ST->hasDQI();
339 bool HasBWI = ST->hasBWI();
340 bool HasVLX = ST->hasVLX();
341
342 struct FixupEntry {
343 int Op;
344 int NumCstElts;
345 int MemBitWidth;
346 std::function<Constant *(const Constant *, unsigned, unsigned, unsigned)>
347 RebuildConstant;
348 };
349 auto FixupConstant = [&](ArrayRef<FixupEntry> Fixups, unsigned RegBitWidth,
350 unsigned OperandNo) {
351#ifdef EXPENSIVE_CHECKS
352 assert(llvm::is_sorted(Fixups,
353 [](const FixupEntry &A, const FixupEntry &B) {
354 return (A.NumCstElts * A.MemBitWidth) <
355 (B.NumCstElts * B.MemBitWidth);
356 }) &&
357 "Constant fixup table not sorted in ascending constant size");
358#endif
359 assert(MI.getNumOperands() >= (OperandNo + X86::AddrNumOperands) &&
360 "Unexpected number of operands!");
361 if (auto *C = X86::getConstantFromPool(MI, OperandNo)) {
362 RegBitWidth =
363 RegBitWidth ? RegBitWidth : C->getType()->getPrimitiveSizeInBits();
364 for (const FixupEntry &Fixup : Fixups) {
365 if (Fixup.Op) {
366 // Construct a suitable constant and adjust the MI to use the new
367 // constant pool entry.
368 if (Constant *NewCst = Fixup.RebuildConstant(
369 C, RegBitWidth, Fixup.NumCstElts, Fixup.MemBitWidth)) {
370 unsigned NewCPI =
371 CP->getConstantPoolIndex(NewCst, Align(Fixup.MemBitWidth / 8));
372 MI.setDesc(TII->get(Fixup.Op));
373 MI.getOperand(OperandNo + X86::AddrDisp).setIndex(NewCPI);
374 return true;
375 }
376 }
377 }
378 }
379 return false;
380 };
381
382 // Attempt to detect a suitable vzload/broadcast/vextload from increasing
383 // constant bitwidths. Prefer vzload/broadcast/vextload for same bitwidth:
384 // - vzload shouldn't ever need a shuffle port to zero the upper elements and
385 // the fp/int domain versions are equally available so we don't introduce a
386 // domain crossing penalty.
387 // - broadcast sometimes need a shuffle port (especially for 8/16-bit
388 // variants), AVX1 only has fp domain broadcasts but AVX2+ have good fp/int
389 // domain equivalents.
390 // - vextload always needs a shuffle port and is only ever int domain.
391 switch (Opc) {
392 /* FP Loads */
393 case X86::MOVAPDrm:
394 case X86::MOVAPSrm:
395 case X86::MOVUPDrm:
396 case X86::MOVUPSrm:
397 // TODO: SSE3 MOVDDUP Handling
398 return FixupConstant({{X86::MOVSSrm, 1, 32, rebuildZeroUpperCst},
399 {X86::MOVSDrm, 1, 64, rebuildZeroUpperCst}},
400 128, 1);
401 case X86::VMOVAPDrm:
402 case X86::VMOVAPSrm:
403 case X86::VMOVUPDrm:
404 case X86::VMOVUPSrm:
405 return FixupConstant({{X86::VMOVSSrm, 1, 32, rebuildZeroUpperCst},
406 {X86::VBROADCASTSSrm, 1, 32, rebuildSplatCst},
407 {X86::VMOVSDrm, 1, 64, rebuildZeroUpperCst},
408 {X86::VMOVDDUPrm, 1, 64, rebuildSplatCst}},
409 128, 1);
410 case X86::VMOVAPDYrm:
411 case X86::VMOVAPSYrm:
412 case X86::VMOVUPDYrm:
413 case X86::VMOVUPSYrm:
414 return FixupConstant({{X86::VBROADCASTSSYrm, 1, 32, rebuildSplatCst},
415 {X86::VBROADCASTSDYrm, 1, 64, rebuildSplatCst},
416 {X86::VBROADCASTF128rm, 1, 128, rebuildSplatCst}},
417 256, 1);
418 case X86::VMOVAPDZ128rm:
419 case X86::VMOVAPSZ128rm:
420 case X86::VMOVUPDZ128rm:
421 case X86::VMOVUPSZ128rm:
422 return FixupConstant({{X86::VMOVSSZrm, 1, 32, rebuildZeroUpperCst},
423 {X86::VBROADCASTSSZ128rm, 1, 32, rebuildSplatCst},
424 {X86::VMOVSDZrm, 1, 64, rebuildZeroUpperCst},
425 {X86::VMOVDDUPZ128rm, 1, 64, rebuildSplatCst}},
426 128, 1);
427 case X86::VMOVAPDZ256rm:
428 case X86::VMOVAPSZ256rm:
429 case X86::VMOVUPDZ256rm:
430 case X86::VMOVUPSZ256rm:
431 return FixupConstant(
432 {{X86::VBROADCASTSSZ256rm, 1, 32, rebuildSplatCst},
433 {X86::VBROADCASTSDZ256rm, 1, 64, rebuildSplatCst},
434 {X86::VBROADCASTF32X4Z256rm, 1, 128, rebuildSplatCst}},
435 256, 1);
436 case X86::VMOVAPDZrm:
437 case X86::VMOVAPSZrm:
438 case X86::VMOVUPDZrm:
439 case X86::VMOVUPSZrm:
440 return FixupConstant({{X86::VBROADCASTSSZrm, 1, 32, rebuildSplatCst},
441 {X86::VBROADCASTSDZrm, 1, 64, rebuildSplatCst},
442 {X86::VBROADCASTF32X4rm, 1, 128, rebuildSplatCst},
443 {X86::VBROADCASTF64X4rm, 1, 256, rebuildSplatCst}},
444 512, 1);
445 /* Integer Loads */
446 case X86::MOVDQArm:
447 case X86::MOVDQUrm: {
448 FixupEntry Fixups[] = {
449 {HasSSE41 ? X86::PMOVSXBQrm : 0, 2, 8, rebuildSExtCst},
450 {HasSSE41 ? X86::PMOVZXBQrm : 0, 2, 8, rebuildZExtCst},
451 {X86::MOVDI2PDIrm, 1, 32, rebuildZeroUpperCst},
452 {HasSSE41 ? X86::PMOVSXBDrm : 0, 4, 8, rebuildSExtCst},
453 {HasSSE41 ? X86::PMOVZXBDrm : 0, 4, 8, rebuildZExtCst},
454 {HasSSE41 ? X86::PMOVSXWQrm : 0, 2, 16, rebuildSExtCst},
455 {HasSSE41 ? X86::PMOVZXWQrm : 0, 2, 16, rebuildZExtCst},
456 {X86::MOVQI2PQIrm, 1, 64, rebuildZeroUpperCst},
457 {HasSSE41 ? X86::PMOVSXBWrm : 0, 8, 8, rebuildSExtCst},
458 {HasSSE41 ? X86::PMOVZXBWrm : 0, 8, 8, rebuildZExtCst},
459 {HasSSE41 ? X86::PMOVSXWDrm : 0, 4, 16, rebuildSExtCst},
460 {HasSSE41 ? X86::PMOVZXWDrm : 0, 4, 16, rebuildZExtCst},
461 {HasSSE41 ? X86::PMOVSXDQrm : 0, 2, 32, rebuildSExtCst},
462 {HasSSE41 ? X86::PMOVZXDQrm : 0, 2, 32, rebuildZExtCst}};
463 return FixupConstant(Fixups, 128, 1);
464 }
465 case X86::VMOVDQArm:
466 case X86::VMOVDQUrm: {
467 FixupEntry Fixups[] = {
468 {HasAVX2 ? X86::VPBROADCASTBrm : 0, 1, 8, rebuildSplatCst},
469 {HasAVX2 ? X86::VPBROADCASTWrm : 0, 1, 16, rebuildSplatCst},
470 {X86::VPMOVSXBQrm, 2, 8, rebuildSExtCst},
471 {X86::VPMOVZXBQrm, 2, 8, rebuildZExtCst},
472 {X86::VMOVDI2PDIrm, 1, 32, rebuildZeroUpperCst},
473 {HasAVX2 ? X86::VPBROADCASTDrm : X86::VBROADCASTSSrm, 1, 32,
475 {X86::VPMOVSXBDrm, 4, 8, rebuildSExtCst},
476 {X86::VPMOVZXBDrm, 4, 8, rebuildZExtCst},
477 {X86::VPMOVSXWQrm, 2, 16, rebuildSExtCst},
478 {X86::VPMOVZXWQrm, 2, 16, rebuildZExtCst},
479 {X86::VMOVQI2PQIrm, 1, 64, rebuildZeroUpperCst},
480 {HasAVX2 ? X86::VPBROADCASTQrm : X86::VMOVDDUPrm, 1, 64,
482 {X86::VPMOVSXBWrm, 8, 8, rebuildSExtCst},
483 {X86::VPMOVZXBWrm, 8, 8, rebuildZExtCst},
484 {X86::VPMOVSXWDrm, 4, 16, rebuildSExtCst},
485 {X86::VPMOVZXWDrm, 4, 16, rebuildZExtCst},
486 {X86::VPMOVSXDQrm, 2, 32, rebuildSExtCst},
487 {X86::VPMOVZXDQrm, 2, 32, rebuildZExtCst}};
488 return FixupConstant(Fixups, 128, 1);
489 }
490 case X86::VMOVDQAYrm:
491 case X86::VMOVDQUYrm: {
492 FixupEntry Fixups[] = {
493 {HasAVX2 ? X86::VPBROADCASTBYrm : 0, 1, 8, rebuildSplatCst},
494 {HasAVX2 ? X86::VPBROADCASTWYrm : 0, 1, 16, rebuildSplatCst},
495 {HasAVX2 ? X86::VPBROADCASTDYrm : X86::VBROADCASTSSYrm, 1, 32,
497 {HasAVX2 ? X86::VPMOVSXBQYrm : 0, 4, 8, rebuildSExtCst},
498 {HasAVX2 ? X86::VPMOVZXBQYrm : 0, 4, 8, rebuildZExtCst},
499 {HasAVX2 ? X86::VPBROADCASTQYrm : X86::VBROADCASTSDYrm, 1, 64,
501 {HasAVX2 ? X86::VPMOVSXBDYrm : 0, 8, 8, rebuildSExtCst},
502 {HasAVX2 ? X86::VPMOVZXBDYrm : 0, 8, 8, rebuildZExtCst},
503 {HasAVX2 ? X86::VPMOVSXWQYrm : 0, 4, 16, rebuildSExtCst},
504 {HasAVX2 ? X86::VPMOVZXWQYrm : 0, 4, 16, rebuildZExtCst},
505 {HasAVX2 ? X86::VBROADCASTI128rm : X86::VBROADCASTF128rm, 1, 128,
507 {HasAVX2 ? X86::VPMOVSXBWYrm : 0, 16, 8, rebuildSExtCst},
508 {HasAVX2 ? X86::VPMOVZXBWYrm : 0, 16, 8, rebuildZExtCst},
509 {HasAVX2 ? X86::VPMOVSXWDYrm : 0, 8, 16, rebuildSExtCst},
510 {HasAVX2 ? X86::VPMOVZXWDYrm : 0, 8, 16, rebuildZExtCst},
511 {HasAVX2 ? X86::VPMOVSXDQYrm : 0, 4, 32, rebuildSExtCst},
512 {HasAVX2 ? X86::VPMOVZXDQYrm : 0, 4, 32, rebuildZExtCst}};
513 return FixupConstant(Fixups, 256, 1);
514 }
515 case X86::VMOVDQA32Z128rm:
516 case X86::VMOVDQA64Z128rm:
517 case X86::VMOVDQU32Z128rm:
518 case X86::VMOVDQU64Z128rm: {
519 FixupEntry Fixups[] = {
520 {HasBWI ? X86::VPBROADCASTBZ128rm : 0, 1, 8, rebuildSplatCst},
521 {HasBWI ? X86::VPBROADCASTWZ128rm : 0, 1, 16, rebuildSplatCst},
522 {X86::VPMOVSXBQZ128rm, 2, 8, rebuildSExtCst},
523 {X86::VPMOVZXBQZ128rm, 2, 8, rebuildZExtCst},
524 {X86::VMOVDI2PDIZrm, 1, 32, rebuildZeroUpperCst},
525 {X86::VPBROADCASTDZ128rm, 1, 32, rebuildSplatCst},
526 {X86::VPMOVSXBDZ128rm, 4, 8, rebuildSExtCst},
527 {X86::VPMOVZXBDZ128rm, 4, 8, rebuildZExtCst},
528 {X86::VPMOVSXWQZ128rm, 2, 16, rebuildSExtCst},
529 {X86::VPMOVZXWQZ128rm, 2, 16, rebuildZExtCst},
530 {X86::VMOVQI2PQIZrm, 1, 64, rebuildZeroUpperCst},
531 {X86::VPBROADCASTQZ128rm, 1, 64, rebuildSplatCst},
532 {HasBWI ? X86::VPMOVSXBWZ128rm : 0, 8, 8, rebuildSExtCst},
533 {HasBWI ? X86::VPMOVZXBWZ128rm : 0, 8, 8, rebuildZExtCst},
534 {X86::VPMOVSXWDZ128rm, 4, 16, rebuildSExtCst},
535 {X86::VPMOVZXWDZ128rm, 4, 16, rebuildZExtCst},
536 {X86::VPMOVSXDQZ128rm, 2, 32, rebuildSExtCst},
537 {X86::VPMOVZXDQZ128rm, 2, 32, rebuildZExtCst}};
538 return FixupConstant(Fixups, 128, 1);
539 }
540 case X86::VMOVDQA32Z256rm:
541 case X86::VMOVDQA64Z256rm:
542 case X86::VMOVDQU32Z256rm:
543 case X86::VMOVDQU64Z256rm: {
544 FixupEntry Fixups[] = {
545 {HasBWI ? X86::VPBROADCASTBZ256rm : 0, 1, 8, rebuildSplatCst},
546 {HasBWI ? X86::VPBROADCASTWZ256rm : 0, 1, 16, rebuildSplatCst},
547 {X86::VPBROADCASTDZ256rm, 1, 32, rebuildSplatCst},
548 {X86::VPMOVSXBQZ256rm, 4, 8, rebuildSExtCst},
549 {X86::VPMOVZXBQZ256rm, 4, 8, rebuildZExtCst},
550 {X86::VPBROADCASTQZ256rm, 1, 64, rebuildSplatCst},
551 {X86::VPMOVSXBDZ256rm, 8, 8, rebuildSExtCst},
552 {X86::VPMOVZXBDZ256rm, 8, 8, rebuildZExtCst},
553 {X86::VPMOVSXWQZ256rm, 4, 16, rebuildSExtCst},
554 {X86::VPMOVZXWQZ256rm, 4, 16, rebuildZExtCst},
555 {X86::VBROADCASTI32X4Z256rm, 1, 128, rebuildSplatCst},
556 {HasBWI ? X86::VPMOVSXBWZ256rm : 0, 16, 8, rebuildSExtCst},
557 {HasBWI ? X86::VPMOVZXBWZ256rm : 0, 16, 8, rebuildZExtCst},
558 {X86::VPMOVSXWDZ256rm, 8, 16, rebuildSExtCst},
559 {X86::VPMOVZXWDZ256rm, 8, 16, rebuildZExtCst},
560 {X86::VPMOVSXDQZ256rm, 4, 32, rebuildSExtCst},
561 {X86::VPMOVZXDQZ256rm, 4, 32, rebuildZExtCst}};
562 return FixupConstant(Fixups, 256, 1);
563 }
564 case X86::VMOVDQA32Zrm:
565 case X86::VMOVDQA64Zrm:
566 case X86::VMOVDQU32Zrm:
567 case X86::VMOVDQU64Zrm: {
568 FixupEntry Fixups[] = {
569 {HasBWI ? X86::VPBROADCASTBZrm : 0, 1, 8, rebuildSplatCst},
570 {HasBWI ? X86::VPBROADCASTWZrm : 0, 1, 16, rebuildSplatCst},
571 {X86::VPBROADCASTDZrm, 1, 32, rebuildSplatCst},
572 {X86::VPBROADCASTQZrm, 1, 64, rebuildSplatCst},
573 {X86::VPMOVSXBQZrm, 8, 8, rebuildSExtCst},
574 {X86::VPMOVZXBQZrm, 8, 8, rebuildZExtCst},
575 {X86::VBROADCASTI32X4rm, 1, 128, rebuildSplatCst},
576 {X86::VPMOVSXBDZrm, 16, 8, rebuildSExtCst},
577 {X86::VPMOVZXBDZrm, 16, 8, rebuildZExtCst},
578 {X86::VPMOVSXWQZrm, 8, 16, rebuildSExtCst},
579 {X86::VPMOVZXWQZrm, 8, 16, rebuildZExtCst},
580 {X86::VBROADCASTI64X4rm, 1, 256, rebuildSplatCst},
581 {HasBWI ? X86::VPMOVSXBWZrm : 0, 32, 8, rebuildSExtCst},
582 {HasBWI ? X86::VPMOVZXBWZrm : 0, 32, 8, rebuildZExtCst},
583 {X86::VPMOVSXWDZrm, 16, 16, rebuildSExtCst},
584 {X86::VPMOVZXWDZrm, 16, 16, rebuildZExtCst},
585 {X86::VPMOVSXDQZrm, 8, 32, rebuildSExtCst},
586 {X86::VPMOVZXDQZrm, 8, 32, rebuildZExtCst}};
587 return FixupConstant(Fixups, 512, 1);
588 }
589 }
590
591 auto ConvertToBroadcastAVX512 = [&](unsigned OpSrc32, unsigned OpSrc64) {
592 unsigned OpBcst32 = 0, OpBcst64 = 0;
593 unsigned OpNoBcst32 = 0, OpNoBcst64 = 0;
594 if (OpSrc32) {
595 if (const X86FoldTableEntry *Mem2Bcst =
597 OpBcst32 = Mem2Bcst->DstOp;
598 OpNoBcst32 = Mem2Bcst->Flags & TB_INDEX_MASK;
599 }
600 }
601 if (OpSrc64) {
602 if (const X86FoldTableEntry *Mem2Bcst =
604 OpBcst64 = Mem2Bcst->DstOp;
605 OpNoBcst64 = Mem2Bcst->Flags & TB_INDEX_MASK;
606 }
607 }
608 assert(((OpBcst32 == 0) || (OpBcst64 == 0) || (OpNoBcst32 == OpNoBcst64)) &&
609 "OperandNo mismatch");
610
611 if (OpBcst32 || OpBcst64) {
612 unsigned OpNo = OpBcst32 == 0 ? OpNoBcst64 : OpNoBcst32;
613 FixupEntry Fixups[] = {{(int)OpBcst32, 32, 32, rebuildSplatCst},
614 {(int)OpBcst64, 64, 64, rebuildSplatCst}};
615 // TODO: Add support for RegBitWidth, but currently rebuildSplatCst
616 // doesn't require it (defaults to Constant::getPrimitiveSizeInBits).
617 return FixupConstant(Fixups, 0, OpNo);
618 }
619 return false;
620 };
621
622 // Attempt to find a AVX512 mapping from a full width memory-fold instruction
623 // to a broadcast-fold instruction variant.
624 if ((MI.getDesc().TSFlags & X86II::EncodingMask) == X86II::EVEX)
625 return ConvertToBroadcastAVX512(Opc, Opc);
626
627 // Reverse the X86InstrInfo::setExecutionDomainCustom EVEX->VEX logic
628 // conversion to see if we can convert to a broadcasted (integer) logic op.
629 if (HasVLX && !HasDQI) {
630 unsigned OpSrc32 = 0, OpSrc64 = 0;
631 switch (Opc) {
632 case X86::VANDPDrm:
633 case X86::VANDPSrm:
634 case X86::VPANDrm:
635 OpSrc32 = X86 ::VPANDDZ128rm;
636 OpSrc64 = X86 ::VPANDQZ128rm;
637 break;
638 case X86::VANDPDYrm:
639 case X86::VANDPSYrm:
640 case X86::VPANDYrm:
641 OpSrc32 = X86 ::VPANDDZ256rm;
642 OpSrc64 = X86 ::VPANDQZ256rm;
643 break;
644 case X86::VANDNPDrm:
645 case X86::VANDNPSrm:
646 case X86::VPANDNrm:
647 OpSrc32 = X86 ::VPANDNDZ128rm;
648 OpSrc64 = X86 ::VPANDNQZ128rm;
649 break;
650 case X86::VANDNPDYrm:
651 case X86::VANDNPSYrm:
652 case X86::VPANDNYrm:
653 OpSrc32 = X86 ::VPANDNDZ256rm;
654 OpSrc64 = X86 ::VPANDNQZ256rm;
655 break;
656 case X86::VORPDrm:
657 case X86::VORPSrm:
658 case X86::VPORrm:
659 OpSrc32 = X86 ::VPORDZ128rm;
660 OpSrc64 = X86 ::VPORQZ128rm;
661 break;
662 case X86::VORPDYrm:
663 case X86::VORPSYrm:
664 case X86::VPORYrm:
665 OpSrc32 = X86 ::VPORDZ256rm;
666 OpSrc64 = X86 ::VPORQZ256rm;
667 break;
668 case X86::VXORPDrm:
669 case X86::VXORPSrm:
670 case X86::VPXORrm:
671 OpSrc32 = X86 ::VPXORDZ128rm;
672 OpSrc64 = X86 ::VPXORQZ128rm;
673 break;
674 case X86::VXORPDYrm:
675 case X86::VXORPSYrm:
676 case X86::VPXORYrm:
677 OpSrc32 = X86 ::VPXORDZ256rm;
678 OpSrc64 = X86 ::VPXORQZ256rm;
679 break;
680 }
681 if (OpSrc32 || OpSrc64)
682 return ConvertToBroadcastAVX512(OpSrc32, OpSrc64);
683 }
684
685 return false;
686}
687
688bool X86FixupVectorConstantsPass::runOnMachineFunction(MachineFunction &MF) {
689 LLVM_DEBUG(dbgs() << "Start X86FixupVectorConstants\n";);
690 bool Changed = false;
692 TII = ST->getInstrInfo();
693 SM = &ST->getSchedModel();
694
695 for (MachineBasicBlock &MBB : MF) {
696 for (MachineInstr &MI : MBB) {
697 if (processInstruction(MF, MBB, MI)) {
698 ++NumInstChanges;
699 Changed = true;
700 }
701 }
702 }
703 LLVM_DEBUG(dbgs() << "End X86FixupVectorConstants\n";);
704 return Changed;
705}
MachineBasicBlock & MBB
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
PowerPC TLS Dynamic Call Fixup
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static Constant * rebuildSplatCst(const Constant *C, unsigned, unsigned, unsigned SplatBitWidth)
static std::optional< APInt > getSplatableConstant(const Constant *C, unsigned SplatBitWidth)
static Constant * rebuildZExtCst(const Constant *C, unsigned NumBits, unsigned NumElts, unsigned SrcEltBitWidth)
static std::optional< APInt > extractConstantBits(const Constant *C)
static Constant * getSplatValueAllowUndef(const ConstantVector *C)
Normally, we only allow poison in vector splats.
static Constant * rebuildExtCst(const Constant *C, bool IsSExt, unsigned NumBits, unsigned NumElts, unsigned SrcEltBitWidth)
static Constant * rebuildZeroUpperCst(const Constant *C, unsigned NumBits, unsigned, unsigned ScalarBitWidth)
static Constant * rebuildSExtCst(const Constant *C, unsigned NumBits, unsigned NumElts, unsigned SrcEltBitWidth)
#define DEBUG_TYPE
static Constant * rebuildConstant(LLVMContext &Ctx, Type *SclTy, const APInt &Bits, unsigned NumSclBits)
Class for arbitrary precision integers.
Definition: APInt.h:78
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1472
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:620
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1491
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:368
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:180
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2966
static Constant * getFP(Type *ElementType, ArrayRef< uint16_t > Elts)
getFP() constructors - Return a constant of vector type with a float element type taken from argument...
Definition: Constants.cpp:3003
Constant Vector Declarations.
Definition: Constants.h:508
This is an important base class in LLVM.
Definition: Constant.h:42
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:432
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Representation of each machine instruction.
Definition: MachineInstr.h:69
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
void push_back(const T &Elt)
Definition: SmallVector.h:427
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:153
bool is16bitFPTy() const
Return true if this is a 16-bit float type.
Definition: Type.h:148
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition: Type.h:156
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:343
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
Definition: X86BaseInfo.h:825
@ AddrNumOperands
Definition: X86BaseInfo.h:36
const Constant * getConstantFromPool(const MachineInstr &MI, unsigned OpNo)
Find any constant pool entry associated with a specific instruction operand.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
const X86FoldTableEntry * lookupBroadcastFoldTableBySize(unsigned MemOp, unsigned BroadcastBits)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
Definition: STLExtras.h:1902
DWARFExpression::Operation Op
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
FunctionPass * createX86FixupVectorConstants()
Return a pass that reduces the size of vector constant pool loads.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Machine model for scheduling, bundling, and heuristics.
Definition: MCSchedule.h:253