LLVM  13.0.0git
X86InstrFMA3Info.cpp
Go to the documentation of this file.
1 //===-- X86InstrFMA3Info.cpp - X86 FMA3 Instruction Information -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the implementation of the classes providing information
10 // about existing X86 FMA3 opcodes, classifying and grouping them.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "X86InstrFMA3Info.h"
15 #include "X86InstrInfo.h"
17 #include "llvm/Support/Threading.h"
18 #include <cassert>
19 #include <cstdint>
20 
21 using namespace llvm;
22 
23 #define FMA3GROUP(Name, Suf, Attrs) \
24  { { X86::Name##132##Suf, X86::Name##213##Suf, X86::Name##231##Suf }, Attrs },
25 
26 #define FMA3GROUP_MASKED(Name, Suf, Attrs) \
27  FMA3GROUP(Name, Suf, Attrs) \
28  FMA3GROUP(Name, Suf##k, Attrs | X86InstrFMA3Group::KMergeMasked) \
29  FMA3GROUP(Name, Suf##kz, Attrs | X86InstrFMA3Group::KZeroMasked)
30 
31 #define FMA3GROUP_PACKED_WIDTHS(Name, Suf, Attrs) \
32  FMA3GROUP(Name, Suf##Ym, Attrs) \
33  FMA3GROUP(Name, Suf##Yr, Attrs) \
34  FMA3GROUP_MASKED(Name, Suf##Z128m, Attrs) \
35  FMA3GROUP_MASKED(Name, Suf##Z128r, Attrs) \
36  FMA3GROUP_MASKED(Name, Suf##Z256m, Attrs) \
37  FMA3GROUP_MASKED(Name, Suf##Z256r, Attrs) \
38  FMA3GROUP_MASKED(Name, Suf##Zm, Attrs) \
39  FMA3GROUP_MASKED(Name, Suf##Zr, Attrs) \
40  FMA3GROUP(Name, Suf##m, Attrs) \
41  FMA3GROUP(Name, Suf##r, Attrs)
42 
43 #define FMA3GROUP_PACKED(Name, Attrs) \
44  FMA3GROUP_PACKED_WIDTHS(Name, PD, Attrs) \
45  FMA3GROUP_PACKED_WIDTHS(Name, PS, Attrs)
46 
47 #define FMA3GROUP_SCALAR_WIDTHS(Name, Suf, Attrs) \
48  FMA3GROUP(Name, Suf##Zm, Attrs) \
49  FMA3GROUP_MASKED(Name, Suf##Zm_Int, Attrs | X86InstrFMA3Group::Intrinsic) \
50  FMA3GROUP(Name, Suf##Zr, Attrs) \
51  FMA3GROUP_MASKED(Name, Suf##Zr_Int, Attrs | X86InstrFMA3Group::Intrinsic) \
52  FMA3GROUP(Name, Suf##m, Attrs) \
53  FMA3GROUP(Name, Suf##m_Int, Attrs | X86InstrFMA3Group::Intrinsic) \
54  FMA3GROUP(Name, Suf##r, Attrs) \
55  FMA3GROUP(Name, Suf##r_Int, Attrs | X86InstrFMA3Group::Intrinsic)
56 
57 #define FMA3GROUP_SCALAR(Name, Attrs) \
58  FMA3GROUP_SCALAR_WIDTHS(Name, SD, Attrs) \
59  FMA3GROUP_SCALAR_WIDTHS(Name, SS, Attrs)
60 
61 #define FMA3GROUP_FULL(Name, Attrs) \
62  FMA3GROUP_PACKED(Name, Attrs) \
63  FMA3GROUP_SCALAR(Name, Attrs)
64 
65 static const X86InstrFMA3Group Groups[] = {
66  FMA3GROUP_FULL(VFMADD, 0)
67  FMA3GROUP_PACKED(VFMADDSUB, 0)
68  FMA3GROUP_FULL(VFMSUB, 0)
69  FMA3GROUP_PACKED(VFMSUBADD, 0)
70  FMA3GROUP_FULL(VFNMADD, 0)
71  FMA3GROUP_FULL(VFNMSUB, 0)
72 };
73 
74 #define FMA3GROUP_PACKED_AVX512_WIDTHS(Name, Type, Suf, Attrs) \
75  FMA3GROUP_MASKED(Name, Type##Z128##Suf, Attrs) \
76  FMA3GROUP_MASKED(Name, Type##Z256##Suf, Attrs) \
77  FMA3GROUP_MASKED(Name, Type##Z##Suf, Attrs)
78 
79 #define FMA3GROUP_PACKED_AVX512(Name, Suf, Attrs) \
80  FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PD, Suf, Attrs) \
81  FMA3GROUP_PACKED_AVX512_WIDTHS(Name, PS, Suf, Attrs)
82 
83 #define FMA3GROUP_PACKED_AVX512_ROUND(Name, Suf, Attrs) \
84  FMA3GROUP_MASKED(Name, PDZ##Suf, Attrs) \
85  FMA3GROUP_MASKED(Name, PSZ##Suf, Attrs)
86 
87 #define FMA3GROUP_SCALAR_AVX512_ROUND(Name, Suf, Attrs) \
88  FMA3GROUP(Name, SDZ##Suf, Attrs) \
89  FMA3GROUP_MASKED(Name, SDZ##Suf##_Int, Attrs) \
90  FMA3GROUP(Name, SSZ##Suf, Attrs) \
91  FMA3GROUP_MASKED(Name, SSZ##Suf##_Int, Attrs)
92 
94  FMA3GROUP_PACKED_AVX512(VFMADD, mb, 0)
95  FMA3GROUP_PACKED_AVX512(VFMADDSUB, mb, 0)
96  FMA3GROUP_PACKED_AVX512(VFMSUB, mb, 0)
97  FMA3GROUP_PACKED_AVX512(VFMSUBADD, mb, 0)
98  FMA3GROUP_PACKED_AVX512(VFNMADD, mb, 0)
99  FMA3GROUP_PACKED_AVX512(VFNMSUB, mb, 0)
100 };
101 
102 static const X86InstrFMA3Group RoundGroups[] = {
103  FMA3GROUP_PACKED_AVX512_ROUND(VFMADD, rb, 0)
105  FMA3GROUP_PACKED_AVX512_ROUND(VFMADDSUB, rb, 0)
106  FMA3GROUP_PACKED_AVX512_ROUND(VFMSUB, rb, 0)
108  FMA3GROUP_PACKED_AVX512_ROUND(VFMSUBADD, rb, 0)
109  FMA3GROUP_PACKED_AVX512_ROUND(VFNMADD, rb, 0)
111  FMA3GROUP_PACKED_AVX512_ROUND(VFNMSUB, rb, 0)
113 };
114 
115 static void verifyTables() {
116 #ifndef NDEBUG
117  static std::atomic<bool> TableChecked(false);
118  if (!TableChecked.load(std::memory_order_relaxed)) {
120  llvm::is_sorted(BroadcastGroups) && "FMA3 tables not sorted!");
121  TableChecked.store(true, std::memory_order_relaxed);
122  }
123 #endif
124 }
125 
126 /// Returns a reference to a group of FMA3 opcodes to where the given
127 /// \p Opcode is included. If the given \p Opcode is not recognized as FMA3
128 /// and not included into any FMA3 group, then nullptr is returned.
129 const X86InstrFMA3Group *llvm::getFMA3Group(unsigned Opcode, uint64_t TSFlags) {
130 
131  // FMA3 instructions have a well defined encoding pattern we can exploit.
132  uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
133  bool IsFMA3 = ((TSFlags & X86II::EncodingMask) == X86II::VEX ||
134  (TSFlags & X86II::EncodingMask) == X86II::EVEX) &&
135  (TSFlags & X86II::OpMapMask) == X86II::T8 &&
136  (TSFlags & X86II::OpPrefixMask) == X86II::PD &&
137  ((BaseOpcode >= 0x96 && BaseOpcode <= 0x9F) ||
138  (BaseOpcode >= 0xA6 && BaseOpcode <= 0xAF) ||
139  (BaseOpcode >= 0xB6 && BaseOpcode <= 0xBF));
140  if (!IsFMA3)
141  return nullptr;
142 
143  verifyTables();
144 
146  if (TSFlags & X86II::EVEX_RC)
147  Table = makeArrayRef(RoundGroups);
148  else if (TSFlags & X86II::EVEX_B)
149  Table = makeArrayRef(BroadcastGroups);
150  else
151  Table = makeArrayRef(Groups);
152 
153  // FMA 132 instructions have an opcode of 0x96-0x9F
154  // FMA 213 instructions have an opcode of 0xA6-0xAF
155  // FMA 231 instructions have an opcode of 0xB6-0xBF
156  unsigned FormIndex = ((BaseOpcode - 0x90) >> 4) & 0x3;
157 
158  auto I = partition_point(Table, [=](const X86InstrFMA3Group &Group) {
159  return Group.Opcodes[FormIndex] < Opcode;
160  });
161  assert(I != Table.end() && I->Opcodes[FormIndex] == Opcode &&
162  "Couldn't find FMA3 opcode!");
163  return I;
164 }
X86InstrFMA3Info.h
llvm
Definition: AllocatorList.h:23
llvm::X86II::PD
@ PD
Definition: X86BaseInfo.h:782
llvm::X86InstrFMA3Group::Intrinsic
@ Intrinsic
This bit must be set in the 'Attributes' field of FMA group if such group of FMA opcodes consists of ...
Definition: X86InstrFMA3Info.h:41
ManagedStatic.h
llvm::X86II::getBaseOpcodeFor
uint8_t getBaseOpcodeFor(uint64_t TSFlags)
Definition: X86BaseInfo.h:976
Groups
static const X86InstrFMA3Group Groups[]
Definition: X86InstrFMA3Info.cpp:65
llvm::X86II::EVEX_B
@ EVEX_B
Definition: X86BaseInfo.h:945
llvm::X86II::OpMapMask
@ OpMapMask
Definition: X86BaseInfo.h:793
llvm::X86II::EncodingMask
@ EncodingMask
Definition: X86BaseInfo.h:896
BroadcastGroups
static const X86InstrFMA3Group BroadcastGroups[]
Definition: X86InstrFMA3Info.cpp:93
FMA3GROUP_FULL
#define FMA3GROUP_FULL(Name, Attrs)
Definition: X86InstrFMA3Info.cpp:61
llvm::X86InstrFMA3Group
This class is used to group {132, 213, 231} forms of FMA opcodes together.
Definition: X86InstrFMA3Info.h:24
FMA3GROUP_SCALAR_AVX512_ROUND
#define FMA3GROUP_SCALAR_AVX512_ROUND(Name, Suf, Attrs)
Definition: X86InstrFMA3Info.cpp:87
verifyTables
static void verifyTables()
Definition: X86InstrFMA3Info.cpp:115
Threading.h
FMA3GROUP_PACKED_AVX512
#define FMA3GROUP_PACKED_AVX512(Name, Suf, Attrs)
Definition: X86InstrFMA3Info.cpp:79
llvm::getFMA3Group
const X86InstrFMA3Group * getFMA3Group(unsigned Opcode, uint64_t TSFlags)
Returns a reference to a group of FMA3 opcodes to where the given Opcode is included.
Definition: X86InstrFMA3Info.cpp:129
I
#define I(x, y, z)
Definition: MD5.cpp:59
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::X86InstrFMA3Group::Opcodes
uint16_t Opcodes[3]
An array holding 3 forms of FMA opcodes.
Definition: X86InstrFMA3Info.h:26
FMA3GROUP_PACKED_AVX512_ROUND
#define FMA3GROUP_PACKED_AVX512_ROUND(Name, Suf, Attrs)
Definition: X86InstrFMA3Info.cpp:83
FMA3GROUP_PACKED
#define FMA3GROUP_PACKED(Name, Attrs)
Definition: X86InstrFMA3Info.cpp:43
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::partition_point
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
Definition: STLExtras.h:1630
RoundGroups
static const X86InstrFMA3Group RoundGroups[]
Definition: X86InstrFMA3Info.cpp:102
llvm::is_sorted
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
Definition: STLExtras.h:1553
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:476
llvm::X86II::EVEX
@ EVEX
Definition: X86BaseInfo.h:908
llvm::X86II::EVEX_RC
@ EVEX_RC
Definition: X86BaseInfo.h:953
llvm::X86II::VEX
@ VEX
Definition: X86BaseInfo.h:899
X86InstrInfo.h
llvm::X86II::OpPrefixMask
@ OpPrefixMask
Definition: X86BaseInfo.h:778
llvm::X86II::T8
@ T8
Definition: X86BaseInfo.h:803
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:154