LLVM 20.0.0git
TargetLoweringBase.cpp
Go to the documentation of this file.
1//===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLoweringBase class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/BitVector.h"
14#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/Loads.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/CallingConv.h"
40#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/Function.h"
43#include "llvm/IR/GlobalValue.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/Type.h"
57#include <algorithm>
58#include <cassert>
59#include <cstdint>
60#include <cstring>
61#include <iterator>
62#include <string>
63#include <tuple>
64#include <utility>
65
66using namespace llvm;
67
69 "jump-is-expensive", cl::init(false),
70 cl::desc("Do not create extra branches to split comparison logic."),
72
74 ("min-jump-table-entries", cl::init(4), cl::Hidden,
75 cl::desc("Set minimum number of entries to use a jump table."));
76
78 ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,
79 cl::desc("Set maximum size of jump tables."));
80
81/// Minimum jump table density for normal functions.
83 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
84 cl::desc("Minimum density for building a jump table in "
85 "a normal function"));
86
87/// Minimum jump table density for -Os or -Oz functions.
89 "optsize-jump-table-density", cl::init(40), cl::Hidden,
90 cl::desc("Minimum density for building a jump table in "
91 "an optsize function"));
92
93// FIXME: This option is only to test if the strict fp operation processed
94// correctly by preventing mutating strict fp operation to normal fp operation
95// during development. When the backend supports strict float operation, this
96// option will be meaningless.
97static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
98 cl::desc("Don't mutate strict-float node to a legalize node"),
99 cl::init(false), cl::Hidden);
100
101/// GetFPLibCall - Helper to return the right libcall for the given floating
102/// point type, or UNKNOWN_LIBCALL if there is none.
104 RTLIB::Libcall Call_F32,
105 RTLIB::Libcall Call_F64,
106 RTLIB::Libcall Call_F80,
107 RTLIB::Libcall Call_F128,
108 RTLIB::Libcall Call_PPCF128) {
109 return
110 VT == MVT::f32 ? Call_F32 :
111 VT == MVT::f64 ? Call_F64 :
112 VT == MVT::f80 ? Call_F80 :
113 VT == MVT::f128 ? Call_F128 :
114 VT == MVT::ppcf128 ? Call_PPCF128 :
115 RTLIB::UNKNOWN_LIBCALL;
116}
117
118/// getFPEXT - Return the FPEXT_*_* value for the given types, or
119/// UNKNOWN_LIBCALL if there is none.
121 if (OpVT == MVT::f16) {
122 if (RetVT == MVT::f32)
123 return FPEXT_F16_F32;
124 if (RetVT == MVT::f64)
125 return FPEXT_F16_F64;
126 if (RetVT == MVT::f80)
127 return FPEXT_F16_F80;
128 if (RetVT == MVT::f128)
129 return FPEXT_F16_F128;
130 } else if (OpVT == MVT::f32) {
131 if (RetVT == MVT::f64)
132 return FPEXT_F32_F64;
133 if (RetVT == MVT::f128)
134 return FPEXT_F32_F128;
135 if (RetVT == MVT::ppcf128)
136 return FPEXT_F32_PPCF128;
137 } else if (OpVT == MVT::f64) {
138 if (RetVT == MVT::f128)
139 return FPEXT_F64_F128;
140 else if (RetVT == MVT::ppcf128)
141 return FPEXT_F64_PPCF128;
142 } else if (OpVT == MVT::f80) {
143 if (RetVT == MVT::f128)
144 return FPEXT_F80_F128;
145 } else if (OpVT == MVT::bf16) {
146 if (RetVT == MVT::f32)
147 return FPEXT_BF16_F32;
148 }
149
150 return UNKNOWN_LIBCALL;
151}
152
153/// getFPROUND - Return the FPROUND_*_* value for the given types, or
154/// UNKNOWN_LIBCALL if there is none.
156 if (RetVT == MVT::f16) {
157 if (OpVT == MVT::f32)
158 return FPROUND_F32_F16;
159 if (OpVT == MVT::f64)
160 return FPROUND_F64_F16;
161 if (OpVT == MVT::f80)
162 return FPROUND_F80_F16;
163 if (OpVT == MVT::f128)
164 return FPROUND_F128_F16;
165 if (OpVT == MVT::ppcf128)
166 return FPROUND_PPCF128_F16;
167 } else if (RetVT == MVT::bf16) {
168 if (OpVT == MVT::f32)
169 return FPROUND_F32_BF16;
170 if (OpVT == MVT::f64)
171 return FPROUND_F64_BF16;
172 } else if (RetVT == MVT::f32) {
173 if (OpVT == MVT::f64)
174 return FPROUND_F64_F32;
175 if (OpVT == MVT::f80)
176 return FPROUND_F80_F32;
177 if (OpVT == MVT::f128)
178 return FPROUND_F128_F32;
179 if (OpVT == MVT::ppcf128)
180 return FPROUND_PPCF128_F32;
181 } else if (RetVT == MVT::f64) {
182 if (OpVT == MVT::f80)
183 return FPROUND_F80_F64;
184 if (OpVT == MVT::f128)
185 return FPROUND_F128_F64;
186 if (OpVT == MVT::ppcf128)
187 return FPROUND_PPCF128_F64;
188 } else if (RetVT == MVT::f80) {
189 if (OpVT == MVT::f128)
190 return FPROUND_F128_F80;
191 }
192
193 return UNKNOWN_LIBCALL;
194}
195
196/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
197/// UNKNOWN_LIBCALL if there is none.
199 if (OpVT == MVT::f16) {
200 if (RetVT == MVT::i32)
201 return FPTOSINT_F16_I32;
202 if (RetVT == MVT::i64)
203 return FPTOSINT_F16_I64;
204 if (RetVT == MVT::i128)
205 return FPTOSINT_F16_I128;
206 } else if (OpVT == MVT::f32) {
207 if (RetVT == MVT::i32)
208 return FPTOSINT_F32_I32;
209 if (RetVT == MVT::i64)
210 return FPTOSINT_F32_I64;
211 if (RetVT == MVT::i128)
212 return FPTOSINT_F32_I128;
213 } else if (OpVT == MVT::f64) {
214 if (RetVT == MVT::i32)
215 return FPTOSINT_F64_I32;
216 if (RetVT == MVT::i64)
217 return FPTOSINT_F64_I64;
218 if (RetVT == MVT::i128)
219 return FPTOSINT_F64_I128;
220 } else if (OpVT == MVT::f80) {
221 if (RetVT == MVT::i32)
222 return FPTOSINT_F80_I32;
223 if (RetVT == MVT::i64)
224 return FPTOSINT_F80_I64;
225 if (RetVT == MVT::i128)
226 return FPTOSINT_F80_I128;
227 } else if (OpVT == MVT::f128) {
228 if (RetVT == MVT::i32)
229 return FPTOSINT_F128_I32;
230 if (RetVT == MVT::i64)
231 return FPTOSINT_F128_I64;
232 if (RetVT == MVT::i128)
233 return FPTOSINT_F128_I128;
234 } else if (OpVT == MVT::ppcf128) {
235 if (RetVT == MVT::i32)
236 return FPTOSINT_PPCF128_I32;
237 if (RetVT == MVT::i64)
238 return FPTOSINT_PPCF128_I64;
239 if (RetVT == MVT::i128)
240 return FPTOSINT_PPCF128_I128;
241 }
242 return UNKNOWN_LIBCALL;
243}
244
245/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
246/// UNKNOWN_LIBCALL if there is none.
248 if (OpVT == MVT::f16) {
249 if (RetVT == MVT::i32)
250 return FPTOUINT_F16_I32;
251 if (RetVT == MVT::i64)
252 return FPTOUINT_F16_I64;
253 if (RetVT == MVT::i128)
254 return FPTOUINT_F16_I128;
255 } else if (OpVT == MVT::f32) {
256 if (RetVT == MVT::i32)
257 return FPTOUINT_F32_I32;
258 if (RetVT == MVT::i64)
259 return FPTOUINT_F32_I64;
260 if (RetVT == MVT::i128)
261 return FPTOUINT_F32_I128;
262 } else if (OpVT == MVT::f64) {
263 if (RetVT == MVT::i32)
264 return FPTOUINT_F64_I32;
265 if (RetVT == MVT::i64)
266 return FPTOUINT_F64_I64;
267 if (RetVT == MVT::i128)
268 return FPTOUINT_F64_I128;
269 } else if (OpVT == MVT::f80) {
270 if (RetVT == MVT::i32)
271 return FPTOUINT_F80_I32;
272 if (RetVT == MVT::i64)
273 return FPTOUINT_F80_I64;
274 if (RetVT == MVT::i128)
275 return FPTOUINT_F80_I128;
276 } else if (OpVT == MVT::f128) {
277 if (RetVT == MVT::i32)
278 return FPTOUINT_F128_I32;
279 if (RetVT == MVT::i64)
280 return FPTOUINT_F128_I64;
281 if (RetVT == MVT::i128)
282 return FPTOUINT_F128_I128;
283 } else if (OpVT == MVT::ppcf128) {
284 if (RetVT == MVT::i32)
285 return FPTOUINT_PPCF128_I32;
286 if (RetVT == MVT::i64)
287 return FPTOUINT_PPCF128_I64;
288 if (RetVT == MVT::i128)
289 return FPTOUINT_PPCF128_I128;
290 }
291 return UNKNOWN_LIBCALL;
292}
293
294/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
295/// UNKNOWN_LIBCALL if there is none.
297 if (OpVT == MVT::i32) {
298 if (RetVT == MVT::f16)
299 return SINTTOFP_I32_F16;
300 if (RetVT == MVT::f32)
301 return SINTTOFP_I32_F32;
302 if (RetVT == MVT::f64)
303 return SINTTOFP_I32_F64;
304 if (RetVT == MVT::f80)
305 return SINTTOFP_I32_F80;
306 if (RetVT == MVT::f128)
307 return SINTTOFP_I32_F128;
308 if (RetVT == MVT::ppcf128)
309 return SINTTOFP_I32_PPCF128;
310 } else if (OpVT == MVT::i64) {
311 if (RetVT == MVT::f16)
312 return SINTTOFP_I64_F16;
313 if (RetVT == MVT::f32)
314 return SINTTOFP_I64_F32;
315 if (RetVT == MVT::f64)
316 return SINTTOFP_I64_F64;
317 if (RetVT == MVT::f80)
318 return SINTTOFP_I64_F80;
319 if (RetVT == MVT::f128)
320 return SINTTOFP_I64_F128;
321 if (RetVT == MVT::ppcf128)
322 return SINTTOFP_I64_PPCF128;
323 } else if (OpVT == MVT::i128) {
324 if (RetVT == MVT::f16)
325 return SINTTOFP_I128_F16;
326 if (RetVT == MVT::f32)
327 return SINTTOFP_I128_F32;
328 if (RetVT == MVT::f64)
329 return SINTTOFP_I128_F64;
330 if (RetVT == MVT::f80)
331 return SINTTOFP_I128_F80;
332 if (RetVT == MVT::f128)
333 return SINTTOFP_I128_F128;
334 if (RetVT == MVT::ppcf128)
335 return SINTTOFP_I128_PPCF128;
336 }
337 return UNKNOWN_LIBCALL;
338}
339
340/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
341/// UNKNOWN_LIBCALL if there is none.
343 if (OpVT == MVT::i32) {
344 if (RetVT == MVT::f16)
345 return UINTTOFP_I32_F16;
346 if (RetVT == MVT::f32)
347 return UINTTOFP_I32_F32;
348 if (RetVT == MVT::f64)
349 return UINTTOFP_I32_F64;
350 if (RetVT == MVT::f80)
351 return UINTTOFP_I32_F80;
352 if (RetVT == MVT::f128)
353 return UINTTOFP_I32_F128;
354 if (RetVT == MVT::ppcf128)
355 return UINTTOFP_I32_PPCF128;
356 } else if (OpVT == MVT::i64) {
357 if (RetVT == MVT::f16)
358 return UINTTOFP_I64_F16;
359 if (RetVT == MVT::f32)
360 return UINTTOFP_I64_F32;
361 if (RetVT == MVT::f64)
362 return UINTTOFP_I64_F64;
363 if (RetVT == MVT::f80)
364 return UINTTOFP_I64_F80;
365 if (RetVT == MVT::f128)
366 return UINTTOFP_I64_F128;
367 if (RetVT == MVT::ppcf128)
368 return UINTTOFP_I64_PPCF128;
369 } else if (OpVT == MVT::i128) {
370 if (RetVT == MVT::f16)
371 return UINTTOFP_I128_F16;
372 if (RetVT == MVT::f32)
373 return UINTTOFP_I128_F32;
374 if (RetVT == MVT::f64)
375 return UINTTOFP_I128_F64;
376 if (RetVT == MVT::f80)
377 return UINTTOFP_I128_F80;
378 if (RetVT == MVT::f128)
379 return UINTTOFP_I128_F128;
380 if (RetVT == MVT::ppcf128)
381 return UINTTOFP_I128_PPCF128;
382 }
383 return UNKNOWN_LIBCALL;
384}
385
387 return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128,
388 POWI_PPCF128);
389}
390
392 return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,
393 LDEXP_PPCF128);
394}
395
397 return getFPLibCall(RetVT, FREXP_F32, FREXP_F64, FREXP_F80, FREXP_F128,
398 FREXP_PPCF128);
399}
400
402 AtomicOrdering Order,
403 uint64_t MemSize) {
404 unsigned ModeN, ModelN;
405 switch (MemSize) {
406 case 1:
407 ModeN = 0;
408 break;
409 case 2:
410 ModeN = 1;
411 break;
412 case 4:
413 ModeN = 2;
414 break;
415 case 8:
416 ModeN = 3;
417 break;
418 case 16:
419 ModeN = 4;
420 break;
421 default:
422 return RTLIB::UNKNOWN_LIBCALL;
423 }
424
425 switch (Order) {
426 case AtomicOrdering::Monotonic:
427 ModelN = 0;
428 break;
429 case AtomicOrdering::Acquire:
430 ModelN = 1;
431 break;
432 case AtomicOrdering::Release:
433 ModelN = 2;
434 break;
435 case AtomicOrdering::AcquireRelease:
436 case AtomicOrdering::SequentiallyConsistent:
437 ModelN = 3;
438 break;
439 default:
440 return UNKNOWN_LIBCALL;
441 }
442
443 return LC[ModeN][ModelN];
444}
445
447 MVT VT) {
448 if (!VT.isScalarInteger())
449 return UNKNOWN_LIBCALL;
450 uint64_t MemSize = VT.getScalarSizeInBits() / 8;
451
452#define LCALLS(A, B) \
453 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
454#define LCALL5(A) \
455 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
456 switch (Opc) {
458 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
459 return getOutlineAtomicHelper(LC, Order, MemSize);
460 }
461 case ISD::ATOMIC_SWAP: {
462 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
463 return getOutlineAtomicHelper(LC, Order, MemSize);
464 }
466 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
467 return getOutlineAtomicHelper(LC, Order, MemSize);
468 }
469 case ISD::ATOMIC_LOAD_OR: {
470 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
471 return getOutlineAtomicHelper(LC, Order, MemSize);
472 }
474 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
475 return getOutlineAtomicHelper(LC, Order, MemSize);
476 }
478 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
479 return getOutlineAtomicHelper(LC, Order, MemSize);
480 }
481 default:
482 return UNKNOWN_LIBCALL;
483 }
484#undef LCALLS
485#undef LCALL5
486}
487
489#define OP_TO_LIBCALL(Name, Enum) \
490 case Name: \
491 switch (VT.SimpleTy) { \
492 default: \
493 return UNKNOWN_LIBCALL; \
494 case MVT::i8: \
495 return Enum##_1; \
496 case MVT::i16: \
497 return Enum##_2; \
498 case MVT::i32: \
499 return Enum##_4; \
500 case MVT::i64: \
501 return Enum##_8; \
502 case MVT::i128: \
503 return Enum##_16; \
504 }
505
506 switch (Opc) {
507 OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
508 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
509 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
510 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
511 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
512 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
513 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
514 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
515 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
516 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
517 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
518 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
519 }
520
521#undef OP_TO_LIBCALL
522
523 return UNKNOWN_LIBCALL;
524}
525
527 switch (ElementSize) {
528 case 1:
529 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
530 case 2:
531 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
532 case 4:
533 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
534 case 8:
535 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
536 case 16:
537 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
538 default:
539 return UNKNOWN_LIBCALL;
540 }
541}
542
544 switch (ElementSize) {
545 case 1:
546 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
547 case 2:
548 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
549 case 4:
550 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
551 case 8:
552 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
553 case 16:
554 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
555 default:
556 return UNKNOWN_LIBCALL;
557 }
558}
559
561 switch (ElementSize) {
562 case 1:
563 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
564 case 2:
565 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
566 case 4:
567 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
568 case 8:
569 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
570 case 16:
571 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
572 default:
573 return UNKNOWN_LIBCALL;
574 }
575}
576
578 std::fill(CmpLibcallCCs, CmpLibcallCCs + RTLIB::UNKNOWN_LIBCALL,
580 CmpLibcallCCs[RTLIB::OEQ_F32] = ISD::SETEQ;
581 CmpLibcallCCs[RTLIB::OEQ_F64] = ISD::SETEQ;
582 CmpLibcallCCs[RTLIB::OEQ_F128] = ISD::SETEQ;
583 CmpLibcallCCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
584 CmpLibcallCCs[RTLIB::UNE_F32] = ISD::SETNE;
585 CmpLibcallCCs[RTLIB::UNE_F64] = ISD::SETNE;
586 CmpLibcallCCs[RTLIB::UNE_F128] = ISD::SETNE;
587 CmpLibcallCCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
588 CmpLibcallCCs[RTLIB::OGE_F32] = ISD::SETGE;
589 CmpLibcallCCs[RTLIB::OGE_F64] = ISD::SETGE;
590 CmpLibcallCCs[RTLIB::OGE_F128] = ISD::SETGE;
591 CmpLibcallCCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
592 CmpLibcallCCs[RTLIB::OLT_F32] = ISD::SETLT;
593 CmpLibcallCCs[RTLIB::OLT_F64] = ISD::SETLT;
594 CmpLibcallCCs[RTLIB::OLT_F128] = ISD::SETLT;
595 CmpLibcallCCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
596 CmpLibcallCCs[RTLIB::OLE_F32] = ISD::SETLE;
597 CmpLibcallCCs[RTLIB::OLE_F64] = ISD::SETLE;
598 CmpLibcallCCs[RTLIB::OLE_F128] = ISD::SETLE;
599 CmpLibcallCCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
600 CmpLibcallCCs[RTLIB::OGT_F32] = ISD::SETGT;
601 CmpLibcallCCs[RTLIB::OGT_F64] = ISD::SETGT;
602 CmpLibcallCCs[RTLIB::OGT_F128] = ISD::SETGT;
603 CmpLibcallCCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
604 CmpLibcallCCs[RTLIB::UO_F32] = ISD::SETNE;
605 CmpLibcallCCs[RTLIB::UO_F64] = ISD::SETNE;
606 CmpLibcallCCs[RTLIB::UO_F128] = ISD::SETNE;
607 CmpLibcallCCs[RTLIB::UO_PPCF128] = ISD::SETNE;
608}
609
610/// NOTE: The TargetMachine owns TLOF.
612 : TM(tm), Libcalls(TM.getTargetTriple()) {
613 initActions();
614
615 // Perform these initializations only once.
621 HasMultipleConditionRegisters = false;
622 HasExtractBitsInsn = false;
623 JumpIsExpensive = JumpIsExpensiveOverride;
625 EnableExtLdPromotion = false;
626 StackPointerRegisterToSaveRestore = 0;
627 BooleanContents = UndefinedBooleanContent;
628 BooleanFloatContents = UndefinedBooleanContent;
629 BooleanVectorContents = UndefinedBooleanContent;
630 SchedPreferenceInfo = Sched::ILP;
633 MaxBytesForAlignment = 0;
634 MaxAtomicSizeInBitsSupported = 0;
635
636 // Assume that even with libcalls, no target supports wider than 128 bit
637 // division.
638 MaxDivRemBitWidthSupported = 128;
639
640 MaxLargeFPConvertBitWidthSupported = llvm::IntegerType::MAX_INT_BITS;
641
642 MinCmpXchgSizeInBits = 0;
643 SupportsUnalignedAtomics = false;
644
645 RTLIB::initCmpLibcallCCs(CmpLibcallCCs);
646}
647
649 // All operations default to being supported.
650 memset(OpActions, 0, sizeof(OpActions));
651 memset(LoadExtActions, 0, sizeof(LoadExtActions));
652 memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
653 memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
654 memset(CondCodeActions, 0, sizeof(CondCodeActions));
655 std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
656 std::fill(std::begin(TargetDAGCombineArray),
657 std::end(TargetDAGCombineArray), 0);
658
659 // Let extending atomic loads be unsupported by default.
660 for (MVT ValVT : MVT::all_valuetypes())
661 for (MVT MemVT : MVT::all_valuetypes())
663 Expand);
664
665 // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to
666 // remove this and targets should individually set these types if not legal.
669 for (MVT VT : {MVT::i2, MVT::i4})
670 OpActions[(unsigned)VT.SimpleTy][NT] = Expand;
671 }
672 for (MVT AVT : MVT::all_valuetypes()) {
673 for (MVT VT : {MVT::i2, MVT::i4, MVT::v128i2, MVT::v64i4}) {
674 setTruncStoreAction(AVT, VT, Expand);
677 }
678 }
679 for (unsigned IM = (unsigned)ISD::PRE_INC;
680 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
681 for (MVT VT : {MVT::i2, MVT::i4}) {
686 }
687 }
688
689 for (MVT VT : MVT::fp_valuetypes()) {
690 MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits());
691 if (IntVT.isValid()) {
694 }
695 }
696
697 // Set default actions for various operations.
698 for (MVT VT : MVT::all_valuetypes()) {
699 // Default all indexed load / store to expand.
700 for (unsigned IM = (unsigned)ISD::PRE_INC;
701 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
706 }
707
708 // Most backends expect to see the node which just returns the value loaded.
710
711 // These operations default to expand.
730 VT, Expand);
731
732 // Overflow operations default to expand
735 VT, Expand);
736
737 // Carry-using overflow operations default to expand.
740 VT, Expand);
741
742 // ADDC/ADDE/SUBC/SUBE default to expand.
744 Expand);
745
746 // [US]CMP default to expand
748
749 // Halving adds
752 Expand);
753
754 // Absolute difference
756
757 // Saturated trunc
761
762 // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
764 Expand);
765
767
768 // These library functions default to expand.
770 Expand);
771
772 // These operations default to expand for vector types.
773 if (VT.isVector())
780 VT, Expand);
781
782 // Constrained floating-point operations default to expand.
783#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
784 setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
785#include "llvm/IR/ConstrainedOps.def"
786
787 // For most targets @llvm.get.dynamic.area.offset just returns 0.
789
790 // Vector reduction default to expand.
798 VT, Expand);
799
800 // Named vector shuffles default to expand.
802
803 // Only some target support this vector operation. Most need to expand it.
805
806 // VP operations default to expand.
807#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
808 setOperationAction(ISD::SDOPC, VT, Expand);
809#include "llvm/IR/VPIntrinsics.def"
810
811 // FP environment operations default to expand.
815 }
816
817 // Most targets ignore the @llvm.prefetch intrinsic.
819
820 // Most targets also ignore the @llvm.readcyclecounter intrinsic.
822
823 // Most targets also ignore the @llvm.readsteadycounter intrinsic.
825
826 // ConstantFP nodes default to expand. Targets can either change this to
827 // Legal, in which case all fp constants are legal, or use isFPImmLegal()
828 // to optimize expansions for certain constants.
830 {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128},
831 Expand);
832
833 // These library functions default to expand.
839 {MVT::f32, MVT::f64, MVT::f128}, Expand);
840
841 // FIXME: Query RuntimeLibCalls to make the decision.
843 {MVT::f32, MVT::f64, MVT::f128}, LibCall);
844
847 MVT::f16, Promote);
848 // Default ISD::TRAP to expand (which turns it into abort).
849 setOperationAction(ISD::TRAP, MVT::Other, Expand);
850
851 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
852 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
854
856
859
860 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
863 }
865
866 // This one by default will call __clear_cache unless the target
867 // wants something different.
869}
870
872 EVT) const {
873 return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
874}
875
877 const DataLayout &DL) const {
878 assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
879 if (LHSTy.isVector())
880 return LHSTy;
881 MVT ShiftVT = getScalarShiftAmountTy(DL, LHSTy);
882 // If any possible shift value won't fit in the prefered type, just use
883 // something safe. Assume it will be legalized when the shift is expanded.
884 if (ShiftVT.getSizeInBits() < Log2_32_Ceil(LHSTy.getSizeInBits()))
885 ShiftVT = MVT::i32;
886 assert(ShiftVT.getSizeInBits() >= Log2_32_Ceil(LHSTy.getSizeInBits()) &&
887 "ShiftVT is still too small!");
888 return ShiftVT;
889}
890
891bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
892 assert(isTypeLegal(VT));
893 switch (Op) {
894 default:
895 return false;
896 case ISD::SDIV:
897 case ISD::UDIV:
898 case ISD::SREM:
899 case ISD::UREM:
900 return true;
901 }
902}
903
905 unsigned DestAS) const {
906 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
907}
908
910 Type *RetTy, ElementCount EC, bool ZeroIsPoison,
911 const ConstantRange *VScaleRange) const {
912 // Find the smallest "sensible" element type to use for the expansion.
913 ConstantRange CR(APInt(64, EC.getKnownMinValue()));
914 if (EC.isScalable())
915 CR = CR.umul_sat(*VScaleRange);
916
917 if (ZeroIsPoison)
918 CR = CR.subtract(APInt(64, 1));
919
920 unsigned EltWidth = RetTy->getScalarSizeInBits();
921 EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
922 EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
923
924 return EltWidth;
925}
926
928 // If the command-line option was specified, ignore this request.
929 if (!JumpIsExpensiveOverride.getNumOccurrences())
930 JumpIsExpensive = isExpensive;
931}
932
935 // If this is a simple type, use the ComputeRegisterProp mechanism.
936 if (VT.isSimple()) {
937 MVT SVT = VT.getSimpleVT();
938 assert((unsigned)SVT.SimpleTy < std::size(TransformToType));
939 MVT NVT = TransformToType[SVT.SimpleTy];
940 LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
941
942 assert((LA == TypeLegal || LA == TypeSoftenFloat ||
943 LA == TypeSoftPromoteHalf ||
944 (NVT.isVector() ||
945 ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) &&
946 "Promote may not follow Expand or Promote");
947
948 if (LA == TypeSplitVector)
949 return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context));
950 if (LA == TypeScalarizeVector)
951 return LegalizeKind(LA, SVT.getVectorElementType());
952 return LegalizeKind(LA, NVT);
953 }
954
955 // Handle Extended Scalar Types.
956 if (!VT.isVector()) {
957 assert(VT.isInteger() && "Float types must be simple");
958 unsigned BitSize = VT.getSizeInBits();
959 // First promote to a power-of-two size, then expand if necessary.
960 if (BitSize < 8 || !isPowerOf2_32(BitSize)) {
961 EVT NVT = VT.getRoundIntegerType(Context);
962 assert(NVT != VT && "Unable to round integer VT");
963 LegalizeKind NextStep = getTypeConversion(Context, NVT);
964 // Avoid multi-step promotion.
965 if (NextStep.first == TypePromoteInteger)
966 return NextStep;
967 // Return rounded integer type.
968 return LegalizeKind(TypePromoteInteger, NVT);
969 }
970
972 EVT::getIntegerVT(Context, VT.getSizeInBits() / 2));
973 }
974
975 // Handle vector types.
976 ElementCount NumElts = VT.getVectorElementCount();
977 EVT EltVT = VT.getVectorElementType();
978
979 // Vectors with only one element are always scalarized.
980 if (NumElts.isScalar())
981 return LegalizeKind(TypeScalarizeVector, EltVT);
982
983 // Try to widen vector elements until the element type is a power of two and
984 // promote it to a legal type later on, for example:
985 // <3 x i8> -> <4 x i8> -> <4 x i32>
986 if (EltVT.isInteger()) {
987 // Vectors with a number of elements that is not a power of two are always
988 // widened, for example <3 x i8> -> <4 x i8>.
989 if (!VT.isPow2VectorType()) {
990 NumElts = NumElts.coefficientNextPowerOf2();
991 EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
992 return LegalizeKind(TypeWidenVector, NVT);
993 }
994
995 // Examine the element type.
996 LegalizeKind LK = getTypeConversion(Context, EltVT);
997
998 // If type is to be expanded, split the vector.
999 // <4 x i140> -> <2 x i140>
1000 if (LK.first == TypeExpandInteger) {
1004 VT.getHalfNumVectorElementsVT(Context));
1005 }
1006
1007 // Promote the integer element types until a legal vector type is found
1008 // or until the element integer type is too big. If a legal type was not
1009 // found, fallback to the usual mechanism of widening/splitting the
1010 // vector.
1011 EVT OldEltVT = EltVT;
1012 while (true) {
1013 // Increase the bitwidth of the element to the next pow-of-two
1014 // (which is greater than 8 bits).
1015 EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
1016 .getRoundIntegerType(Context);
1017
1018 // Stop trying when getting a non-simple element type.
1019 // Note that vector elements may be greater than legal vector element
1020 // types. Example: X86 XMM registers hold 64bit element on 32bit
1021 // systems.
1022 if (!EltVT.isSimple())
1023 break;
1024
1025 // Build a new vector type and check if it is legal.
1026 MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1027 // Found a legal promoted vector type.
1028 if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)
1030 EVT::getVectorVT(Context, EltVT, NumElts));
1031 }
1032
1033 // Reset the type to the unexpanded type if we did not find a legal vector
1034 // type with a promoted vector element type.
1035 EltVT = OldEltVT;
1036 }
1037
1038 // Try to widen the vector until a legal type is found.
1039 // If there is no wider legal type, split the vector.
1040 while (true) {
1041 // Round up to the next power of 2.
1042 NumElts = NumElts.coefficientNextPowerOf2();
1043
1044 // If there is no simple vector type with this many elements then there
1045 // cannot be a larger legal vector type. Note that this assumes that
1046 // there are no skipped intermediate vector types in the simple types.
1047 if (!EltVT.isSimple())
1048 break;
1049 MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1050 if (LargerVector == MVT())
1051 break;
1052
1053 // If this type is legal then widen the vector.
1054 if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
1055 return LegalizeKind(TypeWidenVector, LargerVector);
1056 }
1057
1058 // Widen odd vectors to next power of two.
1059 if (!VT.isPow2VectorType()) {
1060 EVT NVT = VT.getPow2VectorType(Context);
1061 return LegalizeKind(TypeWidenVector, NVT);
1062 }
1063
1066
1067 // Vectors with illegal element types are expanded.
1068 EVT NVT = EVT::getVectorVT(Context, EltVT,
1070 return LegalizeKind(TypeSplitVector, NVT);
1071}
1072
1073static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
1074 unsigned &NumIntermediates,
1075 MVT &RegisterVT,
1076 TargetLoweringBase *TLI) {
1077 // Figure out the right, legal destination reg to copy into.
1079 MVT EltTy = VT.getVectorElementType();
1080
1081 unsigned NumVectorRegs = 1;
1082
1083 // Scalable vectors cannot be scalarized, so splitting or widening is
1084 // required.
1085 if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue()))
1087 "Splitting or widening of non-power-of-2 MVTs is not implemented.");
1088
1089 // FIXME: We don't support non-power-of-2-sized vectors for now.
1090 // Ideally we could break down into LHS/RHS like LegalizeDAG does.
1091 if (!isPowerOf2_32(EC.getKnownMinValue())) {
1092 // Split EC to unit size (scalable property is preserved).
1093 NumVectorRegs = EC.getKnownMinValue();
1094 EC = ElementCount::getFixed(1);
1095 }
1096
1097 // Divide the input until we get to a supported size. This will
1098 // always end up with an EC that represent a scalar or a scalable
1099 // scalar.
1100 while (EC.getKnownMinValue() > 1 &&
1101 !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
1102 EC = EC.divideCoefficientBy(2);
1103 NumVectorRegs <<= 1;
1104 }
1105
1106 NumIntermediates = NumVectorRegs;
1107
1108 MVT NewVT = MVT::getVectorVT(EltTy, EC);
1109 if (!TLI->isTypeLegal(NewVT))
1110 NewVT = EltTy;
1111 IntermediateVT = NewVT;
1112
1113 unsigned LaneSizeInBits = NewVT.getScalarSizeInBits();
1114
1115 // Convert sizes such as i33 to i64.
1116 LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);
1117
1118 MVT DestVT = TLI->getRegisterType(NewVT);
1119 RegisterVT = DestVT;
1120 if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
1121 return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());
1122
1123 // Otherwise, promotion or legal types use the same number of registers as
1124 // the vector decimated to the appropriate level.
1125 return NumVectorRegs;
1126}
1127
1128/// isLegalRC - Return true if the value types that can be represented by the
1129/// specified register class are all legal.
1131 const TargetRegisterClass &RC) const {
1132 for (const auto *I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)
1133 if (isTypeLegal(*I))
1134 return true;
1135 return false;
1136}
1137
1138/// Replace/modify any TargetFrameIndex operands with a targte-dependent
1139/// sequence of memory operands that is recognized by PrologEpilogInserter.
1142 MachineBasicBlock *MBB) const {
1143 MachineInstr *MI = &InitialMI;
1144 MachineFunction &MF = *MI->getMF();
1145 MachineFrameInfo &MFI = MF.getFrameInfo();
1146
1147 // We're handling multiple types of operands here:
1148 // PATCHPOINT MetaArgs - live-in, read only, direct
1149 // STATEPOINT Deopt Spill - live-through, read only, indirect
1150 // STATEPOINT Deopt Alloca - live-through, read only, direct
1151 // (We're currently conservative and mark the deopt slots read/write in
1152 // practice.)
1153 // STATEPOINT GC Spill - live-through, read/write, indirect
1154 // STATEPOINT GC Alloca - live-through, read/write, direct
1155 // The live-in vs live-through is handled already (the live through ones are
1156 // all stack slots), but we need to handle the different type of stackmap
1157 // operands and memory effects here.
1158
1159 if (llvm::none_of(MI->operands(),
1160 [](MachineOperand &Operand) { return Operand.isFI(); }))
1161 return MBB;
1162
1163 MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
1164
1165 // Inherit previous memory operands.
1166 MIB.cloneMemRefs(*MI);
1167
1168 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1169 MachineOperand &MO = MI->getOperand(i);
1170 if (!MO.isFI()) {
1171 // Index of Def operand this Use it tied to.
1172 // Since Defs are coming before Uses, if Use is tied, then
1173 // index of Def must be smaller that index of that Use.
1174 // Also, Defs preserve their position in new MI.
1175 unsigned TiedTo = i;
1176 if (MO.isReg() && MO.isTied())
1177 TiedTo = MI->findTiedOperandIdx(i);
1178 MIB.add(MO);
1179 if (TiedTo < i)
1180 MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);
1181 continue;
1182 }
1183
1184 // foldMemoryOperand builds a new MI after replacing a single FI operand
1185 // with the canonical set of five x86 addressing-mode operands.
1186 int FI = MO.getIndex();
1187
1188 // Add frame index operands recognized by stackmaps.cpp
1190 // indirect-mem-ref tag, size, #FI, offset.
1191 // Used for spills inserted by StatepointLowering. This codepath is not
1192 // used for patchpoints/stackmaps at all, for these spilling is done via
1193 // foldMemoryOperand callback only.
1194 assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
1195 MIB.addImm(StackMaps::IndirectMemRefOp);
1196 MIB.addImm(MFI.getObjectSize(FI));
1197 MIB.add(MO);
1198 MIB.addImm(0);
1199 } else {
1200 // direct-mem-ref tag, #FI, offset.
1201 // Used by patchpoint, and direct alloca arguments to statepoints
1202 MIB.addImm(StackMaps::DirectMemRefOp);
1203 MIB.add(MO);
1204 MIB.addImm(0);
1205 }
1206
1207 assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
1208
1209 // Add a new memory operand for this FI.
1210 assert(MFI.getObjectOffset(FI) != -1);
1211
1212 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and
1213 // PATCHPOINT should be updated to do the same. (TODO)
1214 if (MI->getOpcode() != TargetOpcode::STATEPOINT) {
1215 auto Flags = MachineMemOperand::MOLoad;
1217 MachinePointerInfo::getFixedStack(MF, FI), Flags,
1219 MIB->addMemOperand(MF, MMO);
1220 }
1221 }
1223 MI->eraseFromParent();
1224 return MBB;
1225}
1226
1227/// findRepresentativeClass - Return the largest legal super-reg register class
1228/// of the register class for the specified type and its associated "cost".
1229// This function is in TargetLowering because it uses RegClassForVT which would
1230// need to be moved to TargetRegisterInfo and would necessitate moving
1231// isTypeLegal over as well - a massive change that would just require
1232// TargetLowering having a TargetRegisterInfo class member that it would use.
1233std::pair<const TargetRegisterClass *, uint8_t>
1235 MVT VT) const {
1236 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1237 if (!RC)
1238 return std::make_pair(RC, 0);
1239
1240 // Compute the set of all super-register classes.
1241 BitVector SuperRegRC(TRI->getNumRegClasses());
1242 for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)
1243 SuperRegRC.setBitsInMask(RCI.getMask());
1244
1245 // Find the first legal register class with the largest spill size.
1246 const TargetRegisterClass *BestRC = RC;
1247 for (unsigned i : SuperRegRC.set_bits()) {
1248 const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
1249 // We want the largest possible spill size.
1250 if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
1251 continue;
1252 if (!isLegalRC(*TRI, *SuperRC))
1253 continue;
1254 BestRC = SuperRC;
1255 }
1256 return std::make_pair(BestRC, 1);
1257}
1258
1259/// computeRegisterProperties - Once all of the register classes are added,
1260/// this allows us to compute derived properties we expose.
1262 const TargetRegisterInfo *TRI) {
1263 // Everything defaults to needing one register.
1264 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1265 NumRegistersForVT[i] = 1;
1266 RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
1267 }
1268 // ...except isVoid, which doesn't need any registers.
1269 NumRegistersForVT[MVT::isVoid] = 0;
1270
1271 // Find the largest integer register class.
1272 unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
1273 for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)
1274 assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
1275
1276 // Every integer value type larger than this largest register takes twice as
1277 // many registers to represent as the previous ValueType.
1278 for (unsigned ExpandedReg = LargestIntReg + 1;
1279 ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
1280 NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
1281 RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
1282 TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
1283 ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
1285 }
1286
1287 // Inspect all of the ValueType's smaller than the largest integer
1288 // register to see which ones need promotion.
1289 unsigned LegalIntReg = LargestIntReg;
1290 for (unsigned IntReg = LargestIntReg - 1;
1291 IntReg >= (unsigned)MVT::i1; --IntReg) {
1292 MVT IVT = (MVT::SimpleValueType)IntReg;
1293 if (isTypeLegal(IVT)) {
1294 LegalIntReg = IntReg;
1295 } else {
1296 RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
1297 (MVT::SimpleValueType)LegalIntReg;
1298 ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
1299 }
1300 }
1301
1302 // ppcf128 type is really two f64's.
1303 if (!isTypeLegal(MVT::ppcf128)) {
1304 if (isTypeLegal(MVT::f64)) {
1305 NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
1306 RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
1307 TransformToType[MVT::ppcf128] = MVT::f64;
1308 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
1309 } else {
1310 NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
1311 RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
1312 TransformToType[MVT::ppcf128] = MVT::i128;
1313 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
1314 }
1315 }
1316
1317 // Decide how to handle f128. If the target does not have native f128 support,
1318 // expand it to i128 and we will be generating soft float library calls.
1319 if (!isTypeLegal(MVT::f128)) {
1320 NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
1321 RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
1322 TransformToType[MVT::f128] = MVT::i128;
1323 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
1324 }
1325
1326 // Decide how to handle f80. If the target does not have native f80 support,
1327 // expand it to i96 and we will be generating soft float library calls.
1328 if (!isTypeLegal(MVT::f80)) {
1329 NumRegistersForVT[MVT::f80] = 3*NumRegistersForVT[MVT::i32];
1330 RegisterTypeForVT[MVT::f80] = RegisterTypeForVT[MVT::i32];
1331 TransformToType[MVT::f80] = MVT::i32;
1332 ValueTypeActions.setTypeAction(MVT::f80, TypeSoftenFloat);
1333 }
1334
1335 // Decide how to handle f64. If the target does not have native f64 support,
1336 // expand it to i64 and we will be generating soft float library calls.
1337 if (!isTypeLegal(MVT::f64)) {
1338 NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
1339 RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
1340 TransformToType[MVT::f64] = MVT::i64;
1341 ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
1342 }
1343
1344 // Decide how to handle f32. If the target does not have native f32 support,
1345 // expand it to i32 and we will be generating soft float library calls.
1346 if (!isTypeLegal(MVT::f32)) {
1347 NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
1348 RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
1349 TransformToType[MVT::f32] = MVT::i32;
1350 ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
1351 }
1352
1353 // Decide how to handle f16. If the target does not have native f16 support,
1354 // promote it to f32, because there are no f16 library calls (except for
1355 // conversions).
1356 if (!isTypeLegal(MVT::f16)) {
1357 // Allow targets to control how we legalize half.
1358 bool SoftPromoteHalfType = softPromoteHalfType();
1359 bool UseFPRegsForHalfType = !SoftPromoteHalfType || useFPRegsForHalfType();
1360
1361 if (!UseFPRegsForHalfType) {
1362 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
1363 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
1364 } else {
1365 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
1366 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
1367 }
1368 TransformToType[MVT::f16] = MVT::f32;
1369 if (SoftPromoteHalfType) {
1370 ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf);
1371 } else {
1372 ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
1373 }
1374 }
1375
1376 // Decide how to handle bf16. If the target does not have native bf16 support,
1377 // promote it to f32, because there are no bf16 library calls (except for
1378 // converting from f32 to bf16).
1379 if (!isTypeLegal(MVT::bf16)) {
1380 NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
1381 RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
1382 TransformToType[MVT::bf16] = MVT::f32;
1383 ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
1384 }
1385
1386 // Loop over all of the vector value types to see which need transformations.
1387 for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
1388 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
1389 MVT VT = (MVT::SimpleValueType) i;
1390 if (isTypeLegal(VT))
1391 continue;
1392
1393 MVT EltVT = VT.getVectorElementType();
1395 bool IsLegalWiderType = false;
1396 bool IsScalable = VT.isScalableVector();
1397 LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
1398 switch (PreferredAction) {
1399 case TypePromoteInteger: {
1400 MVT::SimpleValueType EndVT = IsScalable ?
1401 MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE :
1402 MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE;
1403 // Try to promote the elements of integer vectors. If no legal
1404 // promotion was found, fall through to the widen-vector method.
1405 for (unsigned nVT = i + 1;
1406 (MVT::SimpleValueType)nVT <= EndVT; ++nVT) {
1407 MVT SVT = (MVT::SimpleValueType) nVT;
1408 // Promote vectors of integers to vectors with the same number
1409 // of elements, with a wider element type.
1410 if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() &&
1411 SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) {
1412 TransformToType[i] = SVT;
1413 RegisterTypeForVT[i] = SVT;
1414 NumRegistersForVT[i] = 1;
1415 ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
1416 IsLegalWiderType = true;
1417 break;
1418 }
1419 }
1420 if (IsLegalWiderType)
1421 break;
1422 [[fallthrough]];
1423 }
1424
1425 case TypeWidenVector:
1426 if (isPowerOf2_32(EC.getKnownMinValue())) {
1427 // Try to widen the vector.
1428 for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
1429 MVT SVT = (MVT::SimpleValueType) nVT;
1430 if (SVT.getVectorElementType() == EltVT &&
1431 SVT.isScalableVector() == IsScalable &&
1433 EC.getKnownMinValue() &&
1434 isTypeLegal(SVT)) {
1435 TransformToType[i] = SVT;
1436 RegisterTypeForVT[i] = SVT;
1437 NumRegistersForVT[i] = 1;
1438 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1439 IsLegalWiderType = true;
1440 break;
1441 }
1442 }
1443 if (IsLegalWiderType)
1444 break;
1445 } else {
1446 // Only widen to the next power of 2 to keep consistency with EVT.
1447 MVT NVT = VT.getPow2VectorType();
1448 if (isTypeLegal(NVT)) {
1449 TransformToType[i] = NVT;
1450 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1451 RegisterTypeForVT[i] = NVT;
1452 NumRegistersForVT[i] = 1;
1453 break;
1454 }
1455 }
1456 [[fallthrough]];
1457
1458 case TypeSplitVector:
1459 case TypeScalarizeVector: {
1460 MVT IntermediateVT;
1461 MVT RegisterVT;
1462 unsigned NumIntermediates;
1463 unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,
1464 NumIntermediates, RegisterVT, this);
1465 NumRegistersForVT[i] = NumRegisters;
1466 assert(NumRegistersForVT[i] == NumRegisters &&
1467 "NumRegistersForVT size cannot represent NumRegisters!");
1468 RegisterTypeForVT[i] = RegisterVT;
1469
1470 MVT NVT = VT.getPow2VectorType();
1471 if (NVT == VT) {
1472 // Type is already a power of 2. The default action is to split.
1473 TransformToType[i] = MVT::Other;
1474 if (PreferredAction == TypeScalarizeVector)
1475 ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
1476 else if (PreferredAction == TypeSplitVector)
1477 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1478 else if (EC.getKnownMinValue() > 1)
1479 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1480 else
1481 ValueTypeActions.setTypeAction(VT, EC.isScalable()
1484 } else {
1485 TransformToType[i] = NVT;
1486 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1487 }
1488 break;
1489 }
1490 default:
1491 llvm_unreachable("Unknown vector legalization action!");
1492 }
1493 }
1494
1495 // Determine the 'representative' register class for each value type.
1496 // An representative register class is the largest (meaning one which is
1497 // not a sub-register class / subreg register class) legal register class for
1498 // a group of value types. For example, on i386, i8, i16, and i32
1499 // representative would be GR32; while on x86_64 it's GR64.
1500 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1501 const TargetRegisterClass* RRC;
1502 uint8_t Cost;
1504 RepRegClassForVT[i] = RRC;
1505 RepRegClassCostForVT[i] = Cost;
1506 }
1507}
1508
1510 EVT VT) const {
1511 assert(!VT.isVector() && "No default SetCC type for vectors!");
1512 return getPointerTy(DL).SimpleTy;
1513}
1514
1516 return MVT::i32; // return the default value
1517}
1518
1519/// getVectorTypeBreakdown - Vector types are broken down into some number of
1520/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
1521/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
1522/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
1523///
1524/// This method returns the number of registers needed, and the VT for each
1525/// register. It also returns the VT and quantity of the intermediate values
1526/// before they are promoted/expanded.
1528 EVT VT, EVT &IntermediateVT,
1529 unsigned &NumIntermediates,
1530 MVT &RegisterVT) const {
1531 ElementCount EltCnt = VT.getVectorElementCount();
1532
1533 // If there is a wider vector type with the same element type as this one,
1534 // or a promoted vector type that has the same number of elements which
1535 // are wider, then we should convert to that legal vector type.
1536 // This handles things like <2 x float> -> <4 x float> and
1537 // <4 x i1> -> <4 x i32>.
1538 LegalizeTypeAction TA = getTypeAction(Context, VT);
1539 if (!EltCnt.isScalar() &&
1540 (TA == TypeWidenVector || TA == TypePromoteInteger)) {
1541 EVT RegisterEVT = getTypeToTransformTo(Context, VT);
1542 if (isTypeLegal(RegisterEVT)) {
1543 IntermediateVT = RegisterEVT;
1544 RegisterVT = RegisterEVT.getSimpleVT();
1545 NumIntermediates = 1;
1546 return 1;
1547 }
1548 }
1549
1550 // Figure out the right, legal destination reg to copy into.
1551 EVT EltTy = VT.getVectorElementType();
1552
1553 unsigned NumVectorRegs = 1;
1554
1555 // Scalable vectors cannot be scalarized, so handle the legalisation of the
1556 // types like done elsewhere in SelectionDAG.
1557 if (EltCnt.isScalable()) {
1558 LegalizeKind LK;
1559 EVT PartVT = VT;
1560 do {
1561 // Iterate until we've found a legal (part) type to hold VT.
1562 LK = getTypeConversion(Context, PartVT);
1563 PartVT = LK.second;
1564 } while (LK.first != TypeLegal);
1565
1566 if (!PartVT.isVector()) {
1568 "Don't know how to legalize this scalable vector type");
1569 }
1570
1571 NumIntermediates =
1574 IntermediateVT = PartVT;
1575 RegisterVT = getRegisterType(Context, IntermediateVT);
1576 return NumIntermediates;
1577 }
1578
1579 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
1580 // we could break down into LHS/RHS like LegalizeDAG does.
1581 if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
1582 NumVectorRegs = EltCnt.getKnownMinValue();
1583 EltCnt = ElementCount::getFixed(1);
1584 }
1585
1586 // Divide the input until we get to a supported size. This will always
1587 // end with a scalar if the target doesn't support vectors.
1588 while (EltCnt.getKnownMinValue() > 1 &&
1589 !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) {
1590 EltCnt = EltCnt.divideCoefficientBy(2);
1591 NumVectorRegs <<= 1;
1592 }
1593
1594 NumIntermediates = NumVectorRegs;
1595
1596 EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt);
1597 if (!isTypeLegal(NewVT))
1598 NewVT = EltTy;
1599 IntermediateVT = NewVT;
1600
1601 MVT DestVT = getRegisterType(Context, NewVT);
1602 RegisterVT = DestVT;
1603
1604 if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16.
1605 TypeSize NewVTSize = NewVT.getSizeInBits();
1606 // Convert sizes such as i33 to i64.
1607 if (!llvm::has_single_bit<uint32_t>(NewVTSize.getKnownMinValue()))
1608 NewVTSize = NewVTSize.coefficientNextPowerOf2();
1609 return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
1610 }
1611
1612 // Otherwise, promotion or legal types use the same number of registers as
1613 // the vector decimated to the appropriate level.
1614 return NumVectorRegs;
1615}
1616
1618 uint64_t NumCases,
1620 ProfileSummaryInfo *PSI,
1621 BlockFrequencyInfo *BFI) const {
1622 // FIXME: This function check the maximum table size and density, but the
1623 // minimum size is not checked. It would be nice if the minimum size is
1624 // also combined within this function. Currently, the minimum size check is
1625 // performed in findJumpTable() in SelectionDAGBuiler and
1626 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
1627 const bool OptForSize =
1628 SI->getParent()->getParent()->hasOptSize() ||
1629 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);
1630 const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
1631 const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
1632
1633 // Check whether the number of cases is small enough and
1634 // the range is dense enough for a jump table.
1635 return (OptForSize || Range <= MaxJumpTableSize) &&
1636 (NumCases * 100 >= Range * MinDensity);
1637}
1638
1640 EVT ConditionVT) const {
1641 return getRegisterType(Context, ConditionVT);
1642}
1643
1644/// Get the EVTs and ArgFlags collections that represent the legalized return
1645/// type of the given function. This does not require a DAG or a return value,
1646/// and is suitable for use before any DAGs for the function are constructed.
1647/// TODO: Move this out of TargetLowering.cpp.
1649 AttributeList attr,
1651 const TargetLowering &TLI, const DataLayout &DL) {
1652 SmallVector<EVT, 4> ValueVTs;
1653 ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
1654 unsigned NumValues = ValueVTs.size();
1655 if (NumValues == 0) return;
1656
1657 for (unsigned j = 0, f = NumValues; j != f; ++j) {
1658 EVT VT = ValueVTs[j];
1659 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1660
1661 if (attr.hasRetAttr(Attribute::SExt))
1662 ExtendKind = ISD::SIGN_EXTEND;
1663 else if (attr.hasRetAttr(Attribute::ZExt))
1664 ExtendKind = ISD::ZERO_EXTEND;
1665
1666 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1667 VT = TLI.getTypeForExtReturn(ReturnType->getContext(), VT, ExtendKind);
1668
1669 unsigned NumParts =
1670 TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);
1671 MVT PartVT =
1672 TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);
1673
1674 // 'inreg' on function refers to return value
1676 if (attr.hasRetAttr(Attribute::InReg))
1677 Flags.setInReg();
1678
1679 // Propagate extension type if any
1680 if (attr.hasRetAttr(Attribute::SExt))
1681 Flags.setSExt();
1682 else if (attr.hasRetAttr(Attribute::ZExt))
1683 Flags.setZExt();
1684
1685 for (unsigned i = 0; i < NumParts; ++i) {
1686 ISD::ArgFlagsTy OutFlags = Flags;
1687 if (NumParts > 1 && i == 0)
1688 OutFlags.setSplit();
1689 else if (i == NumParts - 1 && i != 0)
1690 OutFlags.setSplitEnd();
1691
1692 Outs.push_back(
1693 ISD::OutputArg(OutFlags, PartVT, VT, /*isfixed=*/true, 0, 0));
1694 }
1695 }
1696}
1697
1698/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1699/// function arguments in the caller parameter area. This is the actual
1700/// alignment, not its logarithm.
1702 const DataLayout &DL) const {
1703 return DL.getABITypeAlign(Ty).value();
1704}
1705
1707 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1708 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
1709 // Check if the specified alignment is sufficient based on the data layout.
1710 // TODO: While using the data layout works in practice, a better solution
1711 // would be to implement this check directly (make this a virtual function).
1712 // For example, the ABI alignment may change based on software platform while
1713 // this function should only be affected by hardware implementation.
1714 Type *Ty = VT.getTypeForEVT(Context);
1715 if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) {
1716 // Assume that an access that meets the ABI-specified alignment is fast.
1717 if (Fast != nullptr)
1718 *Fast = 1;
1719 return true;
1720 }
1721
1722 // This is a misaligned access.
1723 return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
1724}
1725
1727 LLVMContext &Context, const DataLayout &DL, EVT VT,
1728 const MachineMemOperand &MMO, unsigned *Fast) const {
1729 return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),
1730 MMO.getAlign(), MMO.getFlags(), Fast);
1731}
1732
1734 const DataLayout &DL, EVT VT,
1735 unsigned AddrSpace, Align Alignment,
1737 unsigned *Fast) const {
1738 return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
1739 Flags, Fast);
1740}
1741
1743 const DataLayout &DL, EVT VT,
1744 const MachineMemOperand &MMO,
1745 unsigned *Fast) const {
1746 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1747 MMO.getFlags(), Fast);
1748}
1749
1751 const DataLayout &DL, LLT Ty,
1752 const MachineMemOperand &MMO,
1753 unsigned *Fast) const {
1754 EVT VT = getApproximateEVTForLLT(Ty, DL, Context);
1755 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1756 MMO.getFlags(), Fast);
1757}
1758
1759//===----------------------------------------------------------------------===//
1760// TargetTransformInfo Helpers
1761//===----------------------------------------------------------------------===//
1762
1764 enum InstructionOpcodes {
1765#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
1766#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
1767#include "llvm/IR/Instruction.def"
1768 };
1769 switch (static_cast<InstructionOpcodes>(Opcode)) {
1770 case Ret: return 0;
1771 case Br: return 0;
1772 case Switch: return 0;
1773 case IndirectBr: return 0;
1774 case Invoke: return 0;
1775 case CallBr: return 0;
1776 case Resume: return 0;
1777 case Unreachable: return 0;
1778 case CleanupRet: return 0;
1779 case CatchRet: return 0;
1780 case CatchPad: return 0;
1781 case CatchSwitch: return 0;
1782 case CleanupPad: return 0;
1783 case FNeg: return ISD::FNEG;
1784 case Add: return ISD::ADD;
1785 case FAdd: return ISD::FADD;
1786 case Sub: return ISD::SUB;
1787 case FSub: return ISD::FSUB;
1788 case Mul: return ISD::MUL;
1789 case FMul: return ISD::FMUL;
1790 case UDiv: return ISD::UDIV;
1791 case SDiv: return ISD::SDIV;
1792 case FDiv: return ISD::FDIV;
1793 case URem: return ISD::UREM;
1794 case SRem: return ISD::SREM;
1795 case FRem: return ISD::FREM;
1796 case Shl: return ISD::SHL;
1797 case LShr: return ISD::SRL;
1798 case AShr: return ISD::SRA;
1799 case And: return ISD::AND;
1800 case Or: return ISD::OR;
1801 case Xor: return ISD::XOR;
1802 case Alloca: return 0;
1803 case Load: return ISD::LOAD;
1804 case Store: return ISD::STORE;
1805 case GetElementPtr: return 0;
1806 case Fence: return 0;
1807 case AtomicCmpXchg: return 0;
1808 case AtomicRMW: return 0;
1809 case Trunc: return ISD::TRUNCATE;
1810 case ZExt: return ISD::ZERO_EXTEND;
1811 case SExt: return ISD::SIGN_EXTEND;
1812 case FPToUI: return ISD::FP_TO_UINT;
1813 case FPToSI: return ISD::FP_TO_SINT;
1814 case UIToFP: return ISD::UINT_TO_FP;
1815 case SIToFP: return ISD::SINT_TO_FP;
1816 case FPTrunc: return ISD::FP_ROUND;
1817 case FPExt: return ISD::FP_EXTEND;
1818 case PtrToInt: return ISD::BITCAST;
1819 case IntToPtr: return ISD::BITCAST;
1820 case BitCast: return ISD::BITCAST;
1821 case AddrSpaceCast: return ISD::ADDRSPACECAST;
1822 case ICmp: return ISD::SETCC;
1823 case FCmp: return ISD::SETCC;
1824 case PHI: return 0;
1825 case Call: return 0;
1826 case Select: return ISD::SELECT;
1827 case UserOp1: return 0;
1828 case UserOp2: return 0;
1829 case VAArg: return 0;
1830 case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
1831 case InsertElement: return ISD::INSERT_VECTOR_ELT;
1832 case ShuffleVector: return ISD::VECTOR_SHUFFLE;
1833 case ExtractValue: return ISD::MERGE_VALUES;
1834 case InsertValue: return ISD::MERGE_VALUES;
1835 case LandingPad: return 0;
1836 case Freeze: return ISD::FREEZE;
1837 }
1838
1839 llvm_unreachable("Unknown instruction type encountered!");
1840}
1841
1842Value *
1844 bool UseTLS) const {
1845 // compiler-rt provides a variable with a magic name. Targets that do not
1846 // link with compiler-rt may also provide such a variable.
1847 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1848 const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
1849 auto UnsafeStackPtr =
1850 dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
1851
1852 Type *StackPtrTy = PointerType::getUnqual(M->getContext());
1853
1854 if (!UnsafeStackPtr) {
1855 auto TLSModel = UseTLS ?
1858 // The global variable is not defined yet, define it ourselves.
1859 // We use the initial-exec TLS model because we do not support the
1860 // variable living anywhere other than in the main executable.
1861 UnsafeStackPtr = new GlobalVariable(
1862 *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
1863 UnsafeStackPtrVar, nullptr, TLSModel);
1864 } else {
1865 // The variable exists, check its type and attributes.
1866 if (UnsafeStackPtr->getValueType() != StackPtrTy)
1867 report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
1868 if (UseTLS != UnsafeStackPtr->isThreadLocal())
1869 report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
1870 (UseTLS ? "" : "not ") + "be thread-local");
1871 }
1872 return UnsafeStackPtr;
1873}
1874
1875Value *
1877 if (!TM.getTargetTriple().isAndroid())
1878 return getDefaultSafeStackPointerLocation(IRB, true);
1879
1880 // Android provides a libc function to retrieve the address of the current
1881 // thread's unsafe stack pointer.
1882 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1883 auto *PtrTy = PointerType::getUnqual(M->getContext());
1884 FunctionCallee Fn =
1885 M->getOrInsertFunction("__safestack_pointer_address", PtrTy);
1886 return IRB.CreateCall(Fn);
1887}
1888
1889//===----------------------------------------------------------------------===//
1890// Loop Strength Reduction hooks
1891//===----------------------------------------------------------------------===//
1892
1893/// isLegalAddressingMode - Return true if the addressing mode represented
1894/// by AM is legal for this target, for a load/store of the specified type.
1896 const AddrMode &AM, Type *Ty,
1897 unsigned AS, Instruction *I) const {
1898 // The default implementation of this implements a conservative RISCy, r+r and
1899 // r+i addr mode.
1900
1901 // Scalable offsets not supported
1902 if (AM.ScalableOffset)
1903 return false;
1904
1905 // Allows a sign-extended 16-bit immediate field.
1906 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
1907 return false;
1908
1909 // No global is ever allowed as a base.
1910 if (AM.BaseGV)
1911 return false;
1912
1913 // Only support r+r,
1914 switch (AM.Scale) {
1915 case 0: // "r+i" or just "i", depending on HasBaseReg.
1916 break;
1917 case 1:
1918 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
1919 return false;
1920 // Otherwise we have r+r or r+i.
1921 break;
1922 case 2:
1923 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
1924 return false;
1925 // Allow 2*r as r+r.
1926 break;
1927 default: // Don't allow n * r
1928 return false;
1929 }
1930
1931 return true;
1932}
1933
1934//===----------------------------------------------------------------------===//
1935// Stack Protector
1936//===----------------------------------------------------------------------===//
1937
1938// For OpenBSD return its special guard variable. Otherwise return nullptr,
1939// so that SelectionDAG handle SSP.
1941 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
1942 Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
1943 PointerType *PtrTy = PointerType::getUnqual(M.getContext());
1944 Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy);
1945 if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C))
1946 G->setVisibility(GlobalValue::HiddenVisibility);
1947 return C;
1948 }
1949 return nullptr;
1950}
1951
1952// Currently only support "standard" __stack_chk_guard.
1953// TODO: add LOAD_STACK_GUARD support.
1955 if (!M.getNamedValue("__stack_chk_guard")) {
1956 auto *GV = new GlobalVariable(M, PointerType::getUnqual(M.getContext()),
1958 nullptr, "__stack_chk_guard");
1959
1960 // FreeBSD has "__stack_chk_guard" defined externally on libc.so
1961 if (M.getDirectAccessExternalData() &&
1963 !(TM.getTargetTriple().isPPC64() &&
1964 TM.getTargetTriple().isOSFreeBSD()) &&
1965 (!TM.getTargetTriple().isOSDarwin() ||
1967 GV->setDSOLocal(true);
1968 }
1969}
1970
1971// Currently only support "standard" __stack_chk_guard.
1972// TODO: add LOAD_STACK_GUARD support.
1974 return M.getNamedValue("__stack_chk_guard");
1975}
1976
1978 return nullptr;
1979}
1980
1983}
1984
1987}
1988
1989unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
1990 return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;
1991}
1992
1994 return MaximumJumpTableSize;
1995}
1996
1999}
2000
2003}
2004
2006 if (TM.Options.LoopAlignment)
2007 return Align(TM.Options.LoopAlignment);
2008 return PrefLoopAlignment;
2009}
2010
2012 MachineBasicBlock *MBB) const {
2013 return MaxBytesForAlignment;
2014}
2015
2016//===----------------------------------------------------------------------===//
2017// Reciprocal Estimates
2018//===----------------------------------------------------------------------===//
2019
2020/// Get the reciprocal estimate attribute string for a function that will
2021/// override the target defaults.
2023 const Function &F = MF.getFunction();
2024 return F.getFnAttribute("reciprocal-estimates").getValueAsString();
2025}
2026
2027/// Construct a string for the given reciprocal operation of the given type.
2028/// This string should match the corresponding option to the front-end's
2029/// "-mrecip" flag assuming those strings have been passed through in an
2030/// attribute string. For example, "vec-divf" for a division of a vXf32.
2031static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
2032 std::string Name = VT.isVector() ? "vec-" : "";
2033
2034 Name += IsSqrt ? "sqrt" : "div";
2035
2036 // TODO: Handle other float types?
2037 if (VT.getScalarType() == MVT::f64) {
2038 Name += "d";
2039 } else if (VT.getScalarType() == MVT::f16) {
2040 Name += "h";
2041 } else {
2042 assert(VT.getScalarType() == MVT::f32 &&
2043 "Unexpected FP type for reciprocal estimate");
2044 Name += "f";
2045 }
2046
2047 return Name;
2048}
2049
2050/// Return the character position and value (a single numeric character) of a
2051/// customized refinement operation in the input string if it exists. Return
2052/// false if there is no customized refinement step count.
2053static bool parseRefinementStep(StringRef In, size_t &Position,
2054 uint8_t &Value) {
2055 const char RefStepToken = ':';
2056 Position = In.find(RefStepToken);
2057 if (Position == StringRef::npos)
2058 return false;
2059
2060 StringRef RefStepString = In.substr(Position + 1);
2061 // Allow exactly one numeric character for the additional refinement
2062 // step parameter.
2063 if (RefStepString.size() == 1) {
2064 char RefStepChar = RefStepString[0];
2065 if (isDigit(RefStepChar)) {
2066 Value = RefStepChar - '0';
2067 return true;
2068 }
2069 }
2070 report_fatal_error("Invalid refinement step for -recip.");
2071}
2072
2073/// For the input attribute string, return one of the ReciprocalEstimate enum
2074/// status values (enabled, disabled, or not specified) for this operation on
2075/// the specified data type.
2076static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
2077 if (Override.empty())
2079
2080 SmallVector<StringRef, 4> OverrideVector;
2081 Override.split(OverrideVector, ',');
2082 unsigned NumArgs = OverrideVector.size();
2083
2084 // Check if "all", "none", or "default" was specified.
2085 if (NumArgs == 1) {
2086 // Look for an optional setting of the number of refinement steps needed
2087 // for this type of reciprocal operation.
2088 size_t RefPos;
2089 uint8_t RefSteps;
2090 if (parseRefinementStep(Override, RefPos, RefSteps)) {
2091 // Split the string for further processing.
2092 Override = Override.substr(0, RefPos);
2093 }
2094
2095 // All reciprocal types are enabled.
2096 if (Override == "all")
2098
2099 // All reciprocal types are disabled.
2100 if (Override == "none")
2102
2103 // Target defaults for enablement are used.
2104 if (Override == "default")
2106 }
2107
2108 // The attribute string may omit the size suffix ('f'/'d').
2109 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2110 std::string VTNameNoSize = VTName;
2111 VTNameNoSize.pop_back();
2112 static const char DisabledPrefix = '!';
2113
2114 for (StringRef RecipType : OverrideVector) {
2115 size_t RefPos;
2116 uint8_t RefSteps;
2117 if (parseRefinementStep(RecipType, RefPos, RefSteps))
2118 RecipType = RecipType.substr(0, RefPos);
2119
2120 // Ignore the disablement token for string matching.
2121 bool IsDisabled = RecipType[0] == DisabledPrefix;
2122 if (IsDisabled)
2123 RecipType = RecipType.substr(1);
2124
2125 if (RecipType == VTName || RecipType == VTNameNoSize)
2128 }
2129
2131}
2132
2133/// For the input attribute string, return the customized refinement step count
2134/// for this operation on the specified data type. If the step count does not
2135/// exist, return the ReciprocalEstimate enum value for unspecified.
2136static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
2137 if (Override.empty())
2139
2140 SmallVector<StringRef, 4> OverrideVector;
2141 Override.split(OverrideVector, ',');
2142 unsigned NumArgs = OverrideVector.size();
2143
2144 // Check if "all", "default", or "none" was specified.
2145 if (NumArgs == 1) {
2146 // Look for an optional setting of the number of refinement steps needed
2147 // for this type of reciprocal operation.
2148 size_t RefPos;
2149 uint8_t RefSteps;
2150 if (!parseRefinementStep(Override, RefPos, RefSteps))
2152
2153 // Split the string for further processing.
2154 Override = Override.substr(0, RefPos);
2155 assert(Override != "none" &&
2156 "Disabled reciprocals, but specifed refinement steps?");
2157
2158 // If this is a general override, return the specified number of steps.
2159 if (Override == "all" || Override == "default")
2160 return RefSteps;
2161 }
2162
2163 // The attribute string may omit the size suffix ('f'/'d').
2164 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2165 std::string VTNameNoSize = VTName;
2166 VTNameNoSize.pop_back();
2167
2168 for (StringRef RecipType : OverrideVector) {
2169 size_t RefPos;
2170 uint8_t RefSteps;
2171 if (!parseRefinementStep(RecipType, RefPos, RefSteps))
2172 continue;
2173
2174 RecipType = RecipType.substr(0, RefPos);
2175 if (RecipType == VTName || RecipType == VTNameNoSize)
2176 return RefSteps;
2177 }
2178
2180}
2181
2183 MachineFunction &MF) const {
2184 return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
2185}
2186
2188 MachineFunction &MF) const {
2189 return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
2190}
2191
2193 MachineFunction &MF) const {
2194 return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
2195}
2196
2198 MachineFunction &MF) const {
2199 return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
2200}
2201
2203 EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
2204 const MachineMemOperand &MMO) const {
2205 // Single-element vectors are scalarized, so we should generally avoid having
2206 // any memory operations on such types, as they would get scalarized too.
2207 if (LoadVT.isFixedLengthVector() && BitcastVT.isFixedLengthVector() &&
2208 BitcastVT.getVectorNumElements() == 1)
2209 return false;
2210
2211 // Don't do if we could do an indexed load on the original type, but not on
2212 // the new one.
2213 if (!LoadVT.isSimple() || !BitcastVT.isSimple())
2214 return true;
2215
2216 MVT LoadMVT = LoadVT.getSimpleVT();
2217
2218 // Don't bother doing this if it's just going to be promoted again later, as
2219 // doing so might interfere with other combines.
2220 if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
2221 getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
2222 return false;
2223
2224 unsigned Fast = 0;
2225 return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
2226 MMO, &Fast) &&
2227 Fast;
2228}
2229
2232}
2233
2235 const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC,
2236 const TargetLibraryInfo *LibInfo) const {
2238 if (LI.isVolatile())
2240
2241 if (LI.hasMetadata(LLVMContext::MD_nontemporal))
2243
2244 if (LI.hasMetadata(LLVMContext::MD_invariant_load))
2246
2248 LI.getAlign(), DL, &LI, AC,
2249 /*DT=*/nullptr, LibInfo))
2251
2252 Flags |= getTargetMMOFlags(LI);
2253 return Flags;
2254}
2255
2258 const DataLayout &DL) const {
2260
2261 if (SI.isVolatile())
2263
2264 if (SI.hasMetadata(LLVMContext::MD_nontemporal))
2266
2267 // FIXME: Not preserving dereferenceable
2268 Flags |= getTargetMMOFlags(SI);
2269 return Flags;
2270}
2271
2274 const DataLayout &DL) const {
2276
2277 if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {
2278 if (RMW->isVolatile())
2280 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
2281 if (CmpX->isVolatile())
2283 } else
2284 llvm_unreachable("not an atomic instruction");
2285
2286 // FIXME: Not preserving dereferenceable
2287 Flags |= getTargetMMOFlags(AI);
2288 return Flags;
2289}
2290
2292 Instruction *Inst,
2293 AtomicOrdering Ord) const {
2294 if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())
2295 return Builder.CreateFence(Ord);
2296 else
2297 return nullptr;
2298}
2299
2301 Instruction *Inst,
2302 AtomicOrdering Ord) const {
2303 if (isAcquireOrStronger(Ord))
2304 return Builder.CreateFence(Ord);
2305 else
2306 return nullptr;
2307}
2308
2309//===----------------------------------------------------------------------===//
2310// GlobalISel Hooks
2311//===----------------------------------------------------------------------===//
2312
2314 const TargetTransformInfo *TTI) const {
2315 auto &MF = *MI.getMF();
2316 auto &MRI = MF.getRegInfo();
2317 // Assuming a spill and reload of a value has a cost of 1 instruction each,
2318 // this helper function computes the maximum number of uses we should consider
2319 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We
2320 // break even in terms of code size when the original MI has 2 users vs
2321 // choosing to potentially spill. Any more than 2 users we we have a net code
2322 // size increase. This doesn't take into account register pressure though.
2323 auto maxUses = [](unsigned RematCost) {
2324 // A cost of 1 means remats are basically free.
2325 if (RematCost == 1)
2326 return std::numeric_limits<unsigned>::max();
2327 if (RematCost == 2)
2328 return 2U;
2329
2330 // Remat is too expensive, only sink if there's one user.
2331 if (RematCost > 2)
2332 return 1U;
2333 llvm_unreachable("Unexpected remat cost");
2334 };
2335
2336 switch (MI.getOpcode()) {
2337 default:
2338 return false;
2339 // Constants-like instructions should be close to their users.
2340 // We don't want long live-ranges for them.
2341 case TargetOpcode::G_CONSTANT:
2342 case TargetOpcode::G_FCONSTANT:
2343 case TargetOpcode::G_FRAME_INDEX:
2344 case TargetOpcode::G_INTTOPTR:
2345 return true;
2346 case TargetOpcode::G_GLOBAL_VALUE: {
2347 unsigned RematCost = TTI->getGISelRematGlobalCost();
2348 Register Reg = MI.getOperand(0).getReg();
2349 unsigned MaxUses = maxUses(RematCost);
2350 if (MaxUses == UINT_MAX)
2351 return true; // Remats are "free" so always localize.
2352 return MRI.hasAtMostUserInstrs(Reg, MaxUses);
2353 }
2354 }
2355}
unsigned const MachineRegisterInfo * MRI
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
return RetTy
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
static cl::opt< bool > JumpIsExpensiveOverride("jump-is-expensive", cl::init(false), cl::desc("Do not create extra branches to split comparison logic."), cl::Hidden)
#define OP_TO_LIBCALL(Name, Enum)
static cl::opt< unsigned > MinimumJumpTableEntries("min-jump-table-entries", cl::init(4), cl::Hidden, cl::desc("Set minimum number of entries to use a jump table."))
static cl::opt< bool > DisableStrictNodeMutation("disable-strictnode-mutation", cl::desc("Don't mutate strict-float node to a legalize node"), cl::init(false), cl::Hidden)
static bool parseRefinementStep(StringRef In, size_t &Position, uint8_t &Value)
Return the character position and value (a single numeric character) of a customized refinement opera...
static cl::opt< unsigned > MaximumJumpTableSize("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, cl::desc("Set maximum size of jump tables."))
static cl::opt< unsigned > JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, cl::desc("Minimum density for building a jump table in " "a normal function"))
Minimum jump table density for normal functions.
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT, TargetLoweringBase *TLI)
static std::string getReciprocalOpName(bool IsSqrt, EVT VT)
Construct a string for the given reciprocal operation of the given type.
#define LCALL5(A)
static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return the customized refinement step count for this operation on the...
static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return one of the ReciprocalEstimate enum status values (enabled,...
static StringRef getRecipEstimateForFunc(MachineFunction &MF)
Get the reciprocal estimate attribute string for a function that will override the target defaults.
static cl::opt< unsigned > OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden, cl::desc("Minimum density for building a jump table in " "an optsize function"))
Minimum jump table density for -Os or -Oz functions.
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
Class for arbitrary precision integers.
Definition: APInt.h:78
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:495
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:696
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Definition: Attributes.h:820
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:219
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Definition: BitVector.h:707
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:140
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
This class represents a range of values.
Definition: ConstantRange.h:47
unsigned getActiveBits() const
Compute the maximal number of active bits needed to represent every value in this range.
ConstantRange umul_sat(const ConstantRange &Other) const
Perform an unsigned saturating multiplication of two constant ranges.
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size in bytes, rounded up to a whole number of bytes.
Definition: DataLayout.cpp:725
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition: TypeSize.h:314
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:311
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:322
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:68
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:91
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Definition: IRBuilder.h:1851
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:171
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2432
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:363
@ MAX_INT_BITS
Maximum number of bits that can be specified.
Definition: DerivedTypes.h:52
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:174
Value * getPointerOperand()
Definition: Instructions.h:253
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:203
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:209
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getPow2VectorType() const
Widens the length of the given vector MVT up to the nearest power of 2 and returns that type.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isStatepointSpillSlotObjectIndex(int ObjectIdx) const
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:572
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
void freezeReservedRegs()
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Class to represent pointers.
Definition: DerivedTypes.h:646
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
Analysis providing profile information.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:226
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:487
LLVMContext * getContext() const
Definition: SelectionDAG.h:500
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:290
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:685
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:556
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static constexpr size_t npos
Definition: StringRef.h:52
bool isValid() const
Returns true if this iterator is still pointing at a valid entry.
Multiway switch.
Provides information about what library functions are available for the current target.
LegalizeTypeAction getTypeAction(MVT VT) const
void setTypeAction(MVT VT, LegalizeTypeAction Action)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void initActions()
Initialize all of the actions to default values.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const
Returns the target-specific address of the unsafe stack pointer.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
virtual unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
virtual unsigned getMinimumJumpTableEntries() const
Return lower limit for number of blocks in a jump table.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases, uint64_t Range, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const
Return true if lowering to a jump table is suitable for a set of case clusters which may contain NumC...
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual bool useFPRegsForHalfType() const
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
virtual bool softPromoteHalfType() const
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
Value * getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, bool UseTLS) const
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr) const
Return the preferred loop alignment.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool isJumpTableRelative() const
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const
Return the type to use for a scalar shift opcode, given the shifted amount type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
unsigned getMinimumJumpTableDensity(bool OptForSize) const
Return lower limit of the density in a jump table.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
TargetLoweringBase(const TargetMachine &TM)
NOTE: The TargetMachine owns TLOF.
LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const
Return pair that represents the legalization kind (first) that needs to happen to EVT (second) in ord...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
bool isPositionIndependent() const
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
Reloc::Model getRelocationModel() const
Returns the code generation relocation model.
TargetOptions Options
unsigned LoopAlignment
If greater than 0, override TargetLoweringBase::PrefLoopAlignment.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
unsigned getGISelRematGlobalCost() const
bool isWindowsGNUEnvironment() const
Definition: Triple.h:657
bool isAndroid() const
Tests whether the target is Android.
Definition: Triple.h:769
bool isOSFreeBSD() const
Definition: Triple.h:584
bool isPPC64() const
Tests whether the target is 64-bit PowerPC (little and big endian).
Definition: Triple.h:964
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:558
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
constexpr LeafTy coefficientNextPowerOf2() const
Definition: TypeSize.h:262
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:254
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:779
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:752
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:1066
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1407
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1440
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:511
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1337
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:374
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1339
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1340
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:276
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1070
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:501
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1099
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:380
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1089
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:813
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1322
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:840
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:557
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1425
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1429
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:716
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1093
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:870
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1439
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1335
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:953
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1336
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:996
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:387
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1480
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:804
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:684
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1256
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1422
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:751
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1289
@ TRUNCATE_SSAT_U
Definition: ISDOpcodes.h:833
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1426
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:787
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:980
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:334
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1338
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:521
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:356
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:756
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1441
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:641
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:330
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1434
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1084
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1061
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:734
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1334
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:614
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1333
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:1041
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:549
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:810
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1279
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:906
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1316
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1341
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:1028
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1283
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:366
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:338
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:859
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:848
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:696
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:393
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:938
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:310
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1442
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1331
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1047
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1332
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:886
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1250
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:708
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1276
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1075
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:704
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:679
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1423
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:286
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:223
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:538
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:626
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1330
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1001
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:919
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition: ISDOpcodes.h:668
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:881
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:957
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:905
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1430
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:816
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1408
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:507
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:347
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1388
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1080
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
Definition: ISDOpcodes.h:1052
@ TRUNCATE_SSAT_S
TRUNCATE_[SU]SAT_[SU] - Truncate for saturated operand [SU] located in middle, prefix for SAT means i...
Definition: ISDOpcodes.h:831
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition: ISDOpcodes.h:691
@ TRUNCATE_USAT_U
Definition: ISDOpcodes.h:835
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:320
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1603
static const int LAST_INDEXED_MODE
Definition: ISDOpcodes.h:1554
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
void initCmpLibcallCCs(ISD::CondCode *CmpLibcallCCs)
Initialize the default condition code on the libcalls.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getOutlineAtomicHelper(const Libcall(&LC)[5][4], AtomicOrdering Order, uint64_t MemSize)
Return the outline atomics value for the given atomic ordering, access size and set of libcalls for a...
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:353
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
Definition: Sequence.h:337
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
Definition: Loads.cpp:201
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
constexpr force_iteration_on_noniterable_enum_t force_iteration_on_noniterable_enum
Definition: Sequence.h:108
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
bool isReleaseOrStronger(AtomicOrdering AO)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:291
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1736
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition: MathExtras.h:403
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ FMul
Product of floats.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
@ FAdd
Sum of floats.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAcquireOrStronger(AtomicOrdering AO)
InstructionCost Cost
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:35
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
Definition: ValueTypes.h:463
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:137
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:341
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:359
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition: ValueTypes.h:456
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:307
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
bool isFixedLengthVector() const
Definition: ValueTypes.h:178
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
Definition: ValueTypes.h:405
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:314
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:319
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:327
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
Definition: ValueTypes.h:132
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:439
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:152
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...