LLVM 19.0.0git
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the SystemZ target.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SystemZISelLowering.h"
17#include "llvm/Support/Debug.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
25
26namespace {
27// Used to build addressing modes.
28struct SystemZAddressingMode {
29 // The shape of the address.
30 enum AddrForm {
31 // base+displacement
32 FormBD,
33
34 // base+displacement+index for load and store operands
35 FormBDXNormal,
36
37 // base+displacement+index for load address operands
38 FormBDXLA,
39
40 // base+displacement+index+ADJDYNALLOC
41 FormBDXDynAlloc
42 };
43 AddrForm Form;
44
45 // The type of displacement. The enum names here correspond directly
46 // to the definitions in SystemZOperand.td. We could split them into
47 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
48 enum DispRange {
49 Disp12Only,
50 Disp12Pair,
51 Disp20Only,
52 Disp20Only128,
53 Disp20Pair
54 };
55 DispRange DR;
56
57 // The parts of the address. The address is equivalent to:
58 //
59 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
61 int64_t Disp;
63 bool IncludesDynAlloc;
64
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
67
68 // True if the address can have an index register.
69 bool hasIndexField() { return Form != FormBD; }
70
71 // True if the address can (and must) include ADJDYNALLOC.
72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
76
77 errs() << " Base ";
78 if (Base.getNode())
79 Base.getNode()->dump(DAG);
80 else
81 errs() << "null\n";
82
83 if (hasIndexField()) {
84 errs() << " Index ";
85 if (Index.getNode())
86 Index.getNode()->dump(DAG);
87 else
88 errs() << "null\n";
89 }
90
91 errs() << " Disp " << Disp;
92 if (IncludesDynAlloc)
93 errs() << " + ADJDYNALLOC";
94 errs() << '\n';
95 }
96};
97
98// Return a mask with Count low bits set.
99static uint64_t allOnes(unsigned int Count) {
100 assert(Count <= 64);
101 if (Count > 63)
102 return UINT64_MAX;
103 return (uint64_t(1) << Count) - 1;
104}
105
106// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108// Rotate (I5). The combined operand value is effectively:
109//
110// (or (rotl Input, Rotate), ~Mask)
111//
112// for RNSBG and:
113//
114// (and (rotl Input, Rotate), Mask)
115//
116// otherwise. The output value has BitSize bits, although Input may be
117// narrower (in which case the upper bits are don't care), or wider (in which
118// case the result will be truncated as part of the operation).
119struct RxSBGOperands {
120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123 Rotate(0) {}
124
125 unsigned Opcode;
126 unsigned BitSize;
127 uint64_t Mask;
128 SDValue Input;
129 unsigned Start;
130 unsigned End;
131 unsigned Rotate;
132};
133
134class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
136
137 // Used by SystemZOperands.td to create integer constants.
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140 }
141
142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
144 }
145
146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
148 }
149
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
168
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
174
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182 // PC-relative address matching routines used by SystemZOperands.td.
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
186 return true;
187 }
188 return false;
189 }
190
191 // BD matching routines used by SystemZOperands.td.
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194 }
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197 }
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200 }
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203 }
204
205 // MVI matching routines used by SystemZOperands.td.
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208 }
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211 }
212
213 // BDX matching routines used by SystemZOperands.td.
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
219 }
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
225 }
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
231 }
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
237 }
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
243 }
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
249 }
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
255 }
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
261 }
262
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
269
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
272 // set Op to that Y.
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
292
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297 // If Op0 is null, then Node is a constant that can be loaded using:
298 //
299 // (Opcode UpperVal LowerVal)
300 //
301 // If Op0 is nonnull, then Node can be implemented using:
302 //
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
306
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308 SDNode *Node);
309
310 SDNode *loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL);
311
312 // Try to use gather instruction Opcode to implement vector insertion N.
313 bool tryGather(SDNode *N, unsigned Opcode);
314
315 // Try to use scatter instruction Opcode to implement store Store.
316 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
317
318 // Change a chain of {load; op; store} of the same value into a simple op
319 // through memory of that value, if the uses of the modified value and its
320 // address are suitable.
321 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
322
323 // Return true if Load and Store are loads and stores of the same size
324 // and are guaranteed not to overlap. Such operations can be implemented
325 // using block (SS-format) instructions.
326 //
327 // Partial overlap would lead to incorrect code, since the block operations
328 // are logically bytewise, even though they have a fast path for the
329 // non-overlapping case. We also need to avoid full overlap (i.e. two
330 // addresses that might be equal at run time) because although that case
331 // would be handled correctly, it might be implemented by millicode.
332 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
333
334 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
335 // from Y to X.
336 bool storeLoadCanUseMVC(SDNode *N) const;
337
338 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
339 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
340 // to X.
341 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
342
343 // Return true if N (a load or a store) fullfills the alignment
344 // requirements for a PC-relative access.
345 bool storeLoadIsAligned(SDNode *N) const;
346
347 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
348 SDValue expandSelectBoolean(SDNode *Node);
349
350public:
351 static char ID;
352
353 SystemZDAGToDAGISel() = delete;
354
355 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
356 : SelectionDAGISel(ID, TM, OptLevel) {}
357
358 bool runOnMachineFunction(MachineFunction &MF) override {
359 const Function &F = MF.getFunction();
360 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
361 if (F.hasFnAttribute("mnop-mcount"))
362 report_fatal_error("mnop-mcount only supported with fentry-call");
363 if (F.hasFnAttribute("mrecord-mcount"))
364 report_fatal_error("mrecord-mcount only supported with fentry-call");
365 }
366
367 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
369 }
370
371 // Override SelectionDAGISel.
372 void Select(SDNode *Node) override;
374 InlineAsm::ConstraintCode ConstraintID,
375 std::vector<SDValue> &OutOps) override;
376 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
377 void PreprocessISelDAG() override;
378
379 // Include the pieces autogenerated from the target description.
380 #include "SystemZGenDAGISel.inc"
381};
382} // end anonymous namespace
383
384char SystemZDAGToDAGISel::ID = 0;
385
386INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)
387
389 CodeGenOptLevel OptLevel) {
390 return new SystemZDAGToDAGISel(TM, OptLevel);
391}
392
393// Return true if Val should be selected as a displacement for an address
394// with range DR. Here we're interested in the range of both the instruction
395// described by DR and of any pairing instruction.
396static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
397 switch (DR) {
398 case SystemZAddressingMode::Disp12Only:
399 return isUInt<12>(Val);
400
401 case SystemZAddressingMode::Disp12Pair:
402 case SystemZAddressingMode::Disp20Only:
403 case SystemZAddressingMode::Disp20Pair:
404 return isInt<20>(Val);
405
406 case SystemZAddressingMode::Disp20Only128:
407 return isInt<20>(Val) && isInt<20>(Val + 8);
408 }
409 llvm_unreachable("Unhandled displacement range");
410}
411
412// Change the base or index in AM to Value, where IsBase selects
413// between the base and index.
414static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
415 SDValue Value) {
416 if (IsBase)
417 AM.Base = Value;
418 else
419 AM.Index = Value;
420}
421
422// The base or index of AM is equivalent to Value + ADJDYNALLOC,
423// where IsBase selects between the base and index. Try to fold the
424// ADJDYNALLOC into AM.
425static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
426 SDValue Value) {
427 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
428 changeComponent(AM, IsBase, Value);
429 AM.IncludesDynAlloc = true;
430 return true;
431 }
432 return false;
433}
434
435// The base of AM is equivalent to Base + Index. Try to use Index as
436// the index register.
437static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
438 SDValue Index) {
439 if (AM.hasIndexField() && !AM.Index.getNode()) {
440 AM.Base = Base;
441 AM.Index = Index;
442 return true;
443 }
444 return false;
445}
446
447// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
448// between the base and index. Try to fold Op1 into AM's displacement.
449static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
450 SDValue Op0, uint64_t Op1) {
451 // First try adjusting the displacement.
452 int64_t TestDisp = AM.Disp + Op1;
453 if (selectDisp(AM.DR, TestDisp)) {
454 changeComponent(AM, IsBase, Op0);
455 AM.Disp = TestDisp;
456 return true;
457 }
458
459 // We could consider forcing the displacement into a register and
460 // using it as an index, but it would need to be carefully tuned.
461 return false;
462}
463
464bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
465 bool IsBase) const {
466 SDValue N = IsBase ? AM.Base : AM.Index;
467 unsigned Opcode = N.getOpcode();
468 // Look through no-op truncations.
469 if (Opcode == ISD::TRUNCATE && N.getOperand(0).getValueSizeInBits() <= 64) {
470 N = N.getOperand(0);
471 Opcode = N.getOpcode();
472 }
473 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
474 SDValue Op0 = N.getOperand(0);
475 SDValue Op1 = N.getOperand(1);
476
477 unsigned Op0Code = Op0->getOpcode();
478 unsigned Op1Code = Op1->getOpcode();
479
480 if (Op0Code == SystemZISD::ADJDYNALLOC)
481 return expandAdjDynAlloc(AM, IsBase, Op1);
482 if (Op1Code == SystemZISD::ADJDYNALLOC)
483 return expandAdjDynAlloc(AM, IsBase, Op0);
484
485 if (Op0Code == ISD::Constant)
486 return expandDisp(AM, IsBase, Op1,
487 cast<ConstantSDNode>(Op0)->getSExtValue());
488 if (Op1Code == ISD::Constant)
489 return expandDisp(AM, IsBase, Op0,
490 cast<ConstantSDNode>(Op1)->getSExtValue());
491
492 if (IsBase && expandIndex(AM, Op0, Op1))
493 return true;
494 }
495 if (Opcode == SystemZISD::PCREL_OFFSET) {
496 SDValue Full = N.getOperand(0);
497 SDValue Base = N.getOperand(1);
498 SDValue Anchor = Base.getOperand(0);
499 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
500 cast<GlobalAddressSDNode>(Anchor)->getOffset());
501 return expandDisp(AM, IsBase, Base, Offset);
502 }
503 return false;
504}
505
506// Return true if an instruction with displacement range DR should be
507// used for displacement value Val. selectDisp(DR, Val) must already hold.
508static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
509 assert(selectDisp(DR, Val) && "Invalid displacement");
510 switch (DR) {
511 case SystemZAddressingMode::Disp12Only:
512 case SystemZAddressingMode::Disp20Only:
513 case SystemZAddressingMode::Disp20Only128:
514 return true;
515
516 case SystemZAddressingMode::Disp12Pair:
517 // Use the other instruction if the displacement is too large.
518 return isUInt<12>(Val);
519
520 case SystemZAddressingMode::Disp20Pair:
521 // Use the other instruction if the displacement is small enough.
522 return !isUInt<12>(Val);
523 }
524 llvm_unreachable("Unhandled displacement range");
525}
526
527// Return true if Base + Disp + Index should be performed by LA(Y).
528static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
529 // Don't use LA(Y) for constants.
530 if (!Base)
531 return false;
532
533 // Always use LA(Y) for frame addresses, since we know that the destination
534 // register is almost always (perhaps always) going to be different from
535 // the frame register.
536 if (Base->getOpcode() == ISD::FrameIndex)
537 return true;
538
539 if (Disp) {
540 // Always use LA(Y) if there is a base, displacement and index.
541 if (Index)
542 return true;
543
544 // Always use LA if the displacement is small enough. It should always
545 // be no worse than AGHI (and better if it avoids a move).
546 if (isUInt<12>(Disp))
547 return true;
548
549 // For similar reasons, always use LAY if the constant is too big for AGHI.
550 // LAY should be no worse than AGFI.
551 if (!isInt<16>(Disp))
552 return true;
553 } else {
554 // Don't use LA for plain registers.
555 if (!Index)
556 return false;
557
558 // Don't use LA for plain addition if the index operand is only used
559 // once. It should be a natural two-operand addition in that case.
560 if (Index->hasOneUse())
561 return false;
562
563 // Prefer addition if the second operation is sign-extended, in the
564 // hope of using AGF.
565 unsigned IndexOpcode = Index->getOpcode();
566 if (IndexOpcode == ISD::SIGN_EXTEND ||
567 IndexOpcode == ISD::SIGN_EXTEND_INREG)
568 return false;
569 }
570
571 // Don't use LA for two-operand addition if either operand is only
572 // used once. The addition instructions are better in that case.
573 if (Base->hasOneUse())
574 return false;
575
576 return true;
577}
578
579// Return true if Addr is suitable for AM, updating AM if so.
580bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
581 SystemZAddressingMode &AM) const {
582 // Start out assuming that the address will need to be loaded separately,
583 // then try to extend it as much as we can.
584 AM.Base = Addr;
585
586 // First try treating the address as a constant.
587 if (Addr.getOpcode() == ISD::Constant &&
588 expandDisp(AM, true, SDValue(),
589 cast<ConstantSDNode>(Addr)->getSExtValue()))
590 ;
591 // Also see if it's a bare ADJDYNALLOC.
592 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
593 expandAdjDynAlloc(AM, true, SDValue()))
594 ;
595 else
596 // Otherwise try expanding each component.
597 while (expandAddress(AM, true) ||
598 (AM.Index.getNode() && expandAddress(AM, false)))
599 continue;
600
601 // Reject cases where it isn't profitable to use LA(Y).
602 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
603 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
604 return false;
605
606 // Reject cases where the other instruction in a pair should be used.
607 if (!isValidDisp(AM.DR, AM.Disp))
608 return false;
609
610 // Make sure that ADJDYNALLOC is included where necessary.
611 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
612 return false;
613
614 LLVM_DEBUG(AM.dump(CurDAG));
615 return true;
616}
617
618// Insert a node into the DAG at least before Pos. This will reposition
619// the node as needed, and will assign it a node ID that is <= Pos's ID.
620// Note that this does *not* preserve the uniqueness of node IDs!
621// The selection DAG must no longer depend on their uniqueness when this
622// function is used.
623static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
624 if (N->getNodeId() == -1 ||
627 DAG->RepositionNode(Pos->getIterator(), N.getNode());
628 // Mark Node as invalid for pruning as after this it may be a successor to a
629 // selected node but otherwise be in the same position of Pos.
630 // Conservatively mark it with the same -abs(Id) to assure node id
631 // invariant is preserved.
632 N->setNodeId(Pos->getNodeId());
634 }
635}
636
637void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
638 EVT VT, SDValue &Base,
639 SDValue &Disp) const {
640 Base = AM.Base;
641 if (!Base.getNode())
642 // Register 0 means "no base". This is mostly useful for shifts.
643 Base = CurDAG->getRegister(0, VT);
644 else if (Base.getOpcode() == ISD::FrameIndex) {
645 // Lower a FrameIndex to a TargetFrameIndex.
646 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
647 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
648 } else if (Base.getValueType() != VT) {
649 // Truncate values from i64 to i32, for shifts.
650 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
651 "Unexpected truncation");
652 SDLoc DL(Base);
653 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
654 insertDAGNode(CurDAG, Base.getNode(), Trunc);
655 Base = Trunc;
656 }
657
658 // Lower the displacement to a TargetConstant.
659 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
660}
661
662void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
663 EVT VT, SDValue &Base,
664 SDValue &Disp,
665 SDValue &Index) const {
666 getAddressOperands(AM, VT, Base, Disp);
667
668 Index = AM.Index;
669 if (!Index.getNode())
670 // Register 0 means "no index".
671 Index = CurDAG->getRegister(0, VT);
672}
673
674bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
676 SDValue &Disp) const {
677 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
678 if (!selectAddress(Addr, AM))
679 return false;
680
681 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
682 return true;
683}
684
685bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
687 SDValue &Disp) const {
688 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
689 if (!selectAddress(Addr, AM) || AM.Index.getNode())
690 return false;
691
692 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
693 return true;
694}
695
696bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
697 SystemZAddressingMode::DispRange DR,
699 SDValue &Disp, SDValue &Index) const {
700 SystemZAddressingMode AM(Form, DR);
701 if (!selectAddress(Addr, AM))
702 return false;
703
704 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
705 return true;
706}
707
708bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
709 SDValue &Base,
710 SDValue &Disp,
711 SDValue &Index) const {
712 SDValue Regs[2];
713 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
714 Regs[0].getNode() && Regs[1].getNode()) {
715 for (unsigned int I = 0; I < 2; ++I) {
716 Base = Regs[I];
717 Index = Regs[1 - I];
718 // We can't tell here whether the index vector has the right type
719 // for the access; the caller needs to do that instead.
720 if (Index.getOpcode() == ISD::ZERO_EXTEND)
722 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
723 Index.getOperand(1) == Elem) {
725 return true;
726 }
727 }
728 }
729 return false;
730}
731
732bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
733 uint64_t InsertMask) const {
734 // We're only interested in cases where the insertion is into some operand
735 // of Op, rather than into Op itself. The only useful case is an AND.
736 if (Op.getOpcode() != ISD::AND)
737 return false;
738
739 // We need a constant mask.
740 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
741 if (!MaskNode)
742 return false;
743
744 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
745 uint64_t AndMask = MaskNode->getZExtValue();
746 if (InsertMask & AndMask)
747 return false;
748
749 // It's only an insertion if all bits are covered or are known to be zero.
750 // The inner check covers all cases but is more expensive.
751 uint64_t Used = allOnes(Op.getValueSizeInBits());
752 if (Used != (AndMask | InsertMask)) {
753 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
754 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
755 return false;
756 }
757
758 Op = Op.getOperand(0);
759 return true;
760}
761
762bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
763 uint64_t Mask) const {
764 const SystemZInstrInfo *TII = getInstrInfo();
765 if (RxSBG.Rotate != 0)
766 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
767 Mask &= RxSBG.Mask;
768 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
769 RxSBG.Mask = Mask;
770 return true;
771 }
772 return false;
773}
774
775// Return true if any bits of (RxSBG.Input & Mask) are significant.
776static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
777 // Rotate the mask in the same way as RxSBG.Input is rotated.
778 if (RxSBG.Rotate != 0)
779 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
780 return (Mask & RxSBG.Mask) != 0;
781}
782
783bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
784 SDValue N = RxSBG.Input;
785 unsigned Opcode = N.getOpcode();
786 switch (Opcode) {
787 case ISD::TRUNCATE: {
788 if (RxSBG.Opcode == SystemZ::RNSBG)
789 return false;
790 if (N.getOperand(0).getValueSizeInBits() > 64)
791 return false;
792 uint64_t BitSize = N.getValueSizeInBits();
793 uint64_t Mask = allOnes(BitSize);
794 if (!refineRxSBGMask(RxSBG, Mask))
795 return false;
796 RxSBG.Input = N.getOperand(0);
797 return true;
798 }
799 case ISD::AND: {
800 if (RxSBG.Opcode == SystemZ::RNSBG)
801 return false;
802
803 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
804 if (!MaskNode)
805 return false;
806
807 SDValue Input = N.getOperand(0);
808 uint64_t Mask = MaskNode->getZExtValue();
809 if (!refineRxSBGMask(RxSBG, Mask)) {
810 // If some bits of Input are already known zeros, those bits will have
811 // been removed from the mask. See if adding them back in makes the
812 // mask suitable.
813 KnownBits Known = CurDAG->computeKnownBits(Input);
814 Mask |= Known.Zero.getZExtValue();
815 if (!refineRxSBGMask(RxSBG, Mask))
816 return false;
817 }
818 RxSBG.Input = Input;
819 return true;
820 }
821
822 case ISD::OR: {
823 if (RxSBG.Opcode != SystemZ::RNSBG)
824 return false;
825
826 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
827 if (!MaskNode)
828 return false;
829
830 SDValue Input = N.getOperand(0);
831 uint64_t Mask = ~MaskNode->getZExtValue();
832 if (!refineRxSBGMask(RxSBG, Mask)) {
833 // If some bits of Input are already known ones, those bits will have
834 // been removed from the mask. See if adding them back in makes the
835 // mask suitable.
836 KnownBits Known = CurDAG->computeKnownBits(Input);
837 Mask &= ~Known.One.getZExtValue();
838 if (!refineRxSBGMask(RxSBG, Mask))
839 return false;
840 }
841 RxSBG.Input = Input;
842 return true;
843 }
844
845 case ISD::ROTL: {
846 // Any 64-bit rotate left can be merged into the RxSBG.
847 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
848 return false;
849 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
850 if (!CountNode)
851 return false;
852
853 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
854 RxSBG.Input = N.getOperand(0);
855 return true;
856 }
857
858 case ISD::ANY_EXTEND:
859 // Bits above the extended operand are don't-care.
860 RxSBG.Input = N.getOperand(0);
861 return true;
862
863 case ISD::ZERO_EXTEND:
864 if (RxSBG.Opcode != SystemZ::RNSBG) {
865 // Restrict the mask to the extended operand.
866 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
867 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
868 return false;
869
870 RxSBG.Input = N.getOperand(0);
871 return true;
872 }
873 [[fallthrough]];
874
875 case ISD::SIGN_EXTEND: {
876 // Check that the extension bits are don't-care (i.e. are masked out
877 // by the final mask).
878 unsigned BitSize = N.getValueSizeInBits();
879 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
880 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
881 // In the case where only the sign bit is active, increase Rotate with
882 // the extension width.
883 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
884 RxSBG.Rotate += (BitSize - InnerBitSize);
885 else
886 return false;
887 }
888
889 RxSBG.Input = N.getOperand(0);
890 return true;
891 }
892
893 case ISD::SHL: {
894 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
895 if (!CountNode)
896 return false;
897
898 uint64_t Count = CountNode->getZExtValue();
899 unsigned BitSize = N.getValueSizeInBits();
900 if (Count < 1 || Count >= BitSize)
901 return false;
902
903 if (RxSBG.Opcode == SystemZ::RNSBG) {
904 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
905 // count bits from RxSBG.Input are ignored.
906 if (maskMatters(RxSBG, allOnes(Count)))
907 return false;
908 } else {
909 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
910 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
911 return false;
912 }
913
914 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
915 RxSBG.Input = N.getOperand(0);
916 return true;
917 }
918
919 case ISD::SRL:
920 case ISD::SRA: {
921 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
922 if (!CountNode)
923 return false;
924
925 uint64_t Count = CountNode->getZExtValue();
926 unsigned BitSize = N.getValueSizeInBits();
927 if (Count < 1 || Count >= BitSize)
928 return false;
929
930 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
931 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
932 // count bits from RxSBG.Input are ignored.
933 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
934 return false;
935 } else {
936 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
937 // which is similar to SLL above.
938 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
939 return false;
940 }
941
942 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
943 RxSBG.Input = N.getOperand(0);
944 return true;
945 }
946 default:
947 return false;
948 }
949}
950
951SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
952 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
953 return SDValue(N, 0);
954}
955
956SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
957 SDValue N) const {
958 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
959 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
960 DL, VT, getUNDEF(DL, MVT::i64), N);
961 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
962 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
963 assert(N.getValueType() == VT && "Unexpected value types");
964 return N;
965}
966
967bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
968 SDLoc DL(N);
969 EVT VT = N->getValueType(0);
970 if (!VT.isInteger() || VT.getSizeInBits() > 64)
971 return false;
972 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
973 unsigned Count = 0;
974 while (expandRxSBG(RISBG))
975 // The widening or narrowing is expected to be free.
976 // Counting widening or narrowing as a saved operation will result in
977 // preferring an R*SBG over a simple shift/logical instruction.
978 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
979 RISBG.Input.getOpcode() != ISD::TRUNCATE)
980 Count += 1;
981 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
982 return false;
983
984 // Prefer to use normal shift instructions over RISBG, since they can handle
985 // all cases and are sometimes shorter.
986 if (Count == 1 && N->getOpcode() != ISD::AND)
987 return false;
988
989 // Prefer register extensions like LLC over RISBG. Also prefer to start
990 // out with normal ANDs if one instruction would be enough. We can convert
991 // these ANDs into an RISBG later if a three-address instruction is useful.
992 if (RISBG.Rotate == 0) {
993 bool PreferAnd = false;
994 // Prefer AND for any 32-bit and-immediate operation.
995 if (VT == MVT::i32)
996 PreferAnd = true;
997 // As well as for any 64-bit operation that can be implemented via LLC(R),
998 // LLH(R), LLGT(R), or one of the and-immediate instructions.
999 else if (RISBG.Mask == 0xff ||
1000 RISBG.Mask == 0xffff ||
1001 RISBG.Mask == 0x7fffffff ||
1002 SystemZ::isImmLF(~RISBG.Mask) ||
1003 SystemZ::isImmHF(~RISBG.Mask))
1004 PreferAnd = true;
1005 // And likewise for the LLZRGF instruction, which doesn't have a register
1006 // to register version.
1007 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1008 if (Load->getMemoryVT() == MVT::i32 &&
1009 (Load->getExtensionType() == ISD::EXTLOAD ||
1010 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1011 RISBG.Mask == 0xffffff00 &&
1012 Subtarget->hasLoadAndZeroRightmostByte())
1013 PreferAnd = true;
1014 }
1015 if (PreferAnd) {
1016 // Replace the current node with an AND. Note that the current node
1017 // might already be that same AND, in which case it is already CSE'd
1018 // with it, and we must not call ReplaceNode.
1019 SDValue In = convertTo(DL, VT, RISBG.Input);
1020 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1021 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1022 if (N != New.getNode()) {
1023 insertDAGNode(CurDAG, N, Mask);
1024 insertDAGNode(CurDAG, N, New);
1025 ReplaceNode(N, New.getNode());
1026 N = New.getNode();
1027 }
1028 // Now, select the machine opcode to implement this operation.
1029 if (!N->isMachineOpcode())
1030 SelectCode(N);
1031 return true;
1032 }
1033 }
1034
1035 unsigned Opcode = SystemZ::RISBG;
1036 // Prefer RISBGN if available, since it does not clobber CC.
1037 if (Subtarget->hasMiscellaneousExtensions())
1038 Opcode = SystemZ::RISBGN;
1039 EVT OpcodeVT = MVT::i64;
1040 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1041 // We can only use the 32-bit instructions if all source bits are
1042 // in the low 32 bits without wrapping, both after rotation (because
1043 // of the smaller range for Start and End) and before rotation
1044 // (because the input value is truncated).
1045 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1046 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1047 ((RISBG.End + RISBG.Rotate) & 63) >=
1048 ((RISBG.Start + RISBG.Rotate) & 63)) {
1049 Opcode = SystemZ::RISBMux;
1050 OpcodeVT = MVT::i32;
1051 RISBG.Start &= 31;
1052 RISBG.End &= 31;
1053 }
1054 SDValue Ops[5] = {
1055 getUNDEF(DL, OpcodeVT),
1056 convertTo(DL, OpcodeVT, RISBG.Input),
1057 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1058 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1059 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1060 };
1061 SDValue New = convertTo(
1062 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1063 ReplaceNode(N, New.getNode());
1064 return true;
1065}
1066
1067bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1068 SDLoc DL(N);
1069 EVT VT = N->getValueType(0);
1070 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1071 return false;
1072 // Try treating each operand of N as the second operand of the RxSBG
1073 // and see which goes deepest.
1074 RxSBGOperands RxSBG[] = {
1075 RxSBGOperands(Opcode, N->getOperand(0)),
1076 RxSBGOperands(Opcode, N->getOperand(1))
1077 };
1078 unsigned Count[] = { 0, 0 };
1079 for (unsigned I = 0; I < 2; ++I)
1080 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I]))
1081 // In cases of multiple users it seems better to keep the simple
1082 // instruction as they are one cycle faster, and it also helps in cases
1083 // where both inputs share a common node.
1084 // The widening or narrowing is expected to be free. Counting widening
1085 // or narrowing as a saved operation will result in preferring an R*SBG
1086 // over a simple shift/logical instruction.
1087 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1088 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1089 Count[I] += 1;
1090
1091 // Do nothing if neither operand is suitable.
1092 if (Count[0] == 0 && Count[1] == 0)
1093 return false;
1094
1095 // Pick the deepest second operand.
1096 unsigned I = Count[0] > Count[1] ? 0 : 1;
1097 SDValue Op0 = N->getOperand(I ^ 1);
1098
1099 // Prefer IC for character insertions from memory.
1100 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1101 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1102 if (Load->getMemoryVT() == MVT::i8)
1103 return false;
1104
1105 // See whether we can avoid an AND in the first operand by converting
1106 // ROSBG to RISBG.
1107 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1108 Opcode = SystemZ::RISBG;
1109 // Prefer RISBGN if available, since it does not clobber CC.
1110 if (Subtarget->hasMiscellaneousExtensions())
1111 Opcode = SystemZ::RISBGN;
1112 }
1113
1114 SDValue Ops[5] = {
1115 convertTo(DL, MVT::i64, Op0),
1116 convertTo(DL, MVT::i64, RxSBG[I].Input),
1117 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1118 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1119 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1120 };
1121 SDValue New = convertTo(
1122 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1123 ReplaceNode(N, New.getNode());
1124 return true;
1125}
1126
1127void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1128 SDValue Op0, uint64_t UpperVal,
1129 uint64_t LowerVal) {
1130 EVT VT = Node->getValueType(0);
1131 SDLoc DL(Node);
1132 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1133 if (Op0.getNode())
1134 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1135
1136 {
1137 // When we haven't passed in Op0, Upper will be a constant. In order to
1138 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1139 // SelectCode first and end up with an opaque machine node. This means that
1140 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1141 // SelectCode.
1142 //
1143 // Note that in the case where Op0 is passed in we could just call
1144 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1145 // the handle at all, but it's fine to do it here.
1146 //
1147 // TODO: This is a pretty hacky way to do this. Can we do something that
1148 // doesn't require a two paragraph explanation?
1149 HandleSDNode Handle(Upper);
1150 SelectCode(Upper.getNode());
1151 Upper = Handle.getValue();
1152 }
1153
1154 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1155 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1156
1157 ReplaceNode(Node, Or.getNode());
1158
1159 SelectCode(Or.getNode());
1160}
1161
1162void SystemZDAGToDAGISel::loadVectorConstant(
1163 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1167 "Bad opcode!");
1168 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1169 EVT VT = Node->getValueType(0);
1170 SDLoc DL(Node);
1172 for (unsigned OpVal : VCI.OpVals)
1173 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1174 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1175
1176 if (VCI.VecVT == VT.getSimpleVT())
1177 ReplaceNode(Node, Op.getNode());
1178 else if (VT.getSizeInBits() == 128) {
1179 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1180 ReplaceNode(Node, BitCast.getNode());
1181 SelectCode(BitCast.getNode());
1182 } else { // float or double
1183 unsigned SubRegIdx =
1184 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1185 ReplaceNode(
1186 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1187 }
1188 SelectCode(Op.getNode());
1189}
1190
1191SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) {
1192 SDNode *ResNode;
1193 assert (VT.getSizeInBits() == 128);
1194
1195 SDValue CP = CurDAG->getTargetConstantPool(
1196 ConstantInt::get(Type::getInt128Ty(*CurDAG->getContext()), Val),
1197 TLI->getPointerTy(CurDAG->getDataLayout()));
1198
1199 EVT PtrVT = CP.getValueType();
1200 SDValue Ops[] = {
1201 SDValue(CurDAG->getMachineNode(SystemZ::LARL, DL, PtrVT, CP), 0),
1202 CurDAG->getTargetConstant(0, DL, PtrVT),
1203 CurDAG->getRegister(0, PtrVT),
1204 CurDAG->getEntryNode()
1205 };
1206 ResNode = CurDAG->getMachineNode(SystemZ::VL, DL, VT, MVT::Other, Ops);
1207
1208 // Annotate ResNode with memory operand information so that MachineInstr
1209 // queries work properly. This e.g. gives the register allocation the
1210 // required information for rematerialization.
1211 MachineFunction& MF = CurDAG->getMachineFunction();
1215
1216 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
1217 return ResNode;
1218}
1219
1220bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1221 SDValue ElemV = N->getOperand(2);
1222 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1223 if (!ElemN)
1224 return false;
1225
1226 unsigned Elem = ElemN->getZExtValue();
1227 EVT VT = N->getValueType(0);
1228 if (Elem >= VT.getVectorNumElements())
1229 return false;
1230
1231 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1232 if (!Load || !Load->hasNUsesOfValue(1, 0))
1233 return false;
1234 if (Load->getMemoryVT().getSizeInBits() !=
1235 Load->getValueType(0).getSizeInBits())
1236 return false;
1237
1238 SDValue Base, Disp, Index;
1239 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1240 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1241 return false;
1242
1243 SDLoc DL(Load);
1244 SDValue Ops[] = {
1245 N->getOperand(0), Base, Disp, Index,
1246 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1247 };
1248 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1249 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1250 ReplaceNode(N, Res);
1251 return true;
1252}
1253
1254bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1255 SDValue Value = Store->getValue();
1256 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1257 return false;
1258 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1259 return false;
1260
1261 SDValue ElemV = Value.getOperand(1);
1262 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1263 if (!ElemN)
1264 return false;
1265
1266 SDValue Vec = Value.getOperand(0);
1267 EVT VT = Vec.getValueType();
1268 unsigned Elem = ElemN->getZExtValue();
1269 if (Elem >= VT.getVectorNumElements())
1270 return false;
1271
1272 SDValue Base, Disp, Index;
1273 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1274 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1275 return false;
1276
1277 SDLoc DL(Store);
1278 SDValue Ops[] = {
1279 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1280 Store->getChain()
1281 };
1282 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1283 return true;
1284}
1285
1286// Check whether or not the chain ending in StoreNode is suitable for doing
1287// the {load; op; store} to modify transformation.
1289 SDValue StoredVal, SelectionDAG *CurDAG,
1290 LoadSDNode *&LoadNode,
1291 SDValue &InputChain) {
1292 // Is the stored value result 0 of the operation?
1293 if (StoredVal.getResNo() != 0)
1294 return false;
1295
1296 // Are there other uses of the loaded value than the operation?
1297 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1298 return false;
1299
1300 // Is the store non-extending and non-indexed?
1301 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1302 return false;
1303
1304 SDValue Load = StoredVal->getOperand(0);
1305 // Is the stored value a non-extending and non-indexed load?
1306 if (!ISD::isNormalLoad(Load.getNode()))
1307 return false;
1308
1309 // Return LoadNode by reference.
1310 LoadNode = cast<LoadSDNode>(Load);
1311
1312 // Is store the only read of the loaded value?
1313 if (!Load.hasOneUse())
1314 return false;
1315
1316 // Is the address of the store the same as the load?
1317 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1318 LoadNode->getOffset() != StoreNode->getOffset())
1319 return false;
1320
1321 // Check if the chain is produced by the load or is a TokenFactor with
1322 // the load output chain as an operand. Return InputChain by reference.
1323 SDValue Chain = StoreNode->getChain();
1324
1325 bool ChainCheck = false;
1326 if (Chain == Load.getValue(1)) {
1327 ChainCheck = true;
1328 InputChain = LoadNode->getChain();
1329 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1330 SmallVector<SDValue, 4> ChainOps;
1331 SmallVector<const SDNode *, 4> LoopWorklist;
1333 const unsigned int Max = 1024;
1334 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1335 SDValue Op = Chain.getOperand(i);
1336 if (Op == Load.getValue(1)) {
1337 ChainCheck = true;
1338 // Drop Load, but keep its chain. No cycle check necessary.
1339 ChainOps.push_back(Load.getOperand(0));
1340 continue;
1341 }
1342 LoopWorklist.push_back(Op.getNode());
1343 ChainOps.push_back(Op);
1344 }
1345
1346 if (ChainCheck) {
1347 // Add the other operand of StoredVal to worklist.
1348 for (SDValue Op : StoredVal->ops())
1349 if (Op.getNode() != LoadNode)
1350 LoopWorklist.push_back(Op.getNode());
1351
1352 // Check if Load is reachable from any of the nodes in the worklist.
1353 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1354 true))
1355 return false;
1356
1357 // Make a new TokenFactor with all the other input chains except
1358 // for the load.
1359 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1360 MVT::Other, ChainOps);
1361 }
1362 }
1363 if (!ChainCheck)
1364 return false;
1365
1366 return true;
1367}
1368
1369// Change a chain of {load; op; store} of the same value into a simple op
1370// through memory of that value, if the uses of the modified value and its
1371// address are suitable.
1372//
1373// The tablegen pattern memory operand pattern is currently not able to match
1374// the case where the CC on the original operation are used.
1375//
1376// See the equivalent routine in X86ISelDAGToDAG for further comments.
1377bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1378 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1379 SDValue StoredVal = StoreNode->getOperand(1);
1380 unsigned Opc = StoredVal->getOpcode();
1381 SDLoc DL(StoreNode);
1382
1383 // Before we try to select anything, make sure this is memory operand size
1384 // and opcode we can handle. Note that this must match the code below that
1385 // actually lowers the opcodes.
1386 EVT MemVT = StoreNode->getMemoryVT();
1387 unsigned NewOpc = 0;
1388 bool NegateOperand = false;
1389 switch (Opc) {
1390 default:
1391 return false;
1392 case SystemZISD::SSUBO:
1393 NegateOperand = true;
1394 [[fallthrough]];
1395 case SystemZISD::SADDO:
1396 if (MemVT == MVT::i32)
1397 NewOpc = SystemZ::ASI;
1398 else if (MemVT == MVT::i64)
1399 NewOpc = SystemZ::AGSI;
1400 else
1401 return false;
1402 break;
1403 case SystemZISD::USUBO:
1404 NegateOperand = true;
1405 [[fallthrough]];
1406 case SystemZISD::UADDO:
1407 if (MemVT == MVT::i32)
1408 NewOpc = SystemZ::ALSI;
1409 else if (MemVT == MVT::i64)
1410 NewOpc = SystemZ::ALGSI;
1411 else
1412 return false;
1413 break;
1414 }
1415
1416 LoadSDNode *LoadNode = nullptr;
1417 SDValue InputChain;
1418 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1419 InputChain))
1420 return false;
1421
1422 SDValue Operand = StoredVal.getOperand(1);
1423 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1424 if (!OperandC)
1425 return false;
1426 auto OperandV = OperandC->getAPIntValue();
1427 if (NegateOperand)
1428 OperandV = -OperandV;
1429 if (OperandV.getSignificantBits() > 8)
1430 return false;
1431 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1432
1433 SDValue Base, Disp;
1434 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1435 return false;
1436
1437 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1439 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1440 CurDAG->setNodeMemRefs(
1441 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1442
1443 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1444 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1445 CurDAG->RemoveDeadNode(Node);
1446 return true;
1447}
1448
1449bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1450 LoadSDNode *Load) const {
1451 // Check that the two memory operands have the same size.
1452 if (Load->getMemoryVT() != Store->getMemoryVT())
1453 return false;
1454
1455 // Volatility stops an access from being decomposed.
1456 if (Load->isVolatile() || Store->isVolatile())
1457 return false;
1458
1459 // There's no chance of overlap if the load is invariant.
1460 if (Load->isInvariant() && Load->isDereferenceable())
1461 return true;
1462
1463 // Otherwise we need to check whether there's an alias.
1464 const Value *V1 = Load->getMemOperand()->getValue();
1465 const Value *V2 = Store->getMemOperand()->getValue();
1466 if (!V1 || !V2)
1467 return false;
1468
1469 // Reject equality.
1470 uint64_t Size = Load->getMemoryVT().getStoreSize();
1471 int64_t End1 = Load->getSrcValueOffset() + Size;
1472 int64_t End2 = Store->getSrcValueOffset() + Size;
1473 if (V1 == V2 && End1 == End2)
1474 return false;
1475
1476 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1477 MemoryLocation(V2, End2, Store->getAAInfo()));
1478}
1479
1480bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1481 auto *Store = cast<StoreSDNode>(N);
1482 auto *Load = cast<LoadSDNode>(Store->getValue());
1483
1484 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1485 // instructions.
1486 uint64_t Size = Load->getMemoryVT().getStoreSize();
1487 if (Size > 1 && Size <= 8) {
1488 // Prefer LHRL, LRL and LGRL.
1489 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1490 return false;
1491 // Prefer STHRL, STRL and STGRL.
1492 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1493 return false;
1494 }
1495
1496 return canUseBlockOperation(Store, Load);
1497}
1498
1499bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1500 unsigned I) const {
1501 auto *StoreA = cast<StoreSDNode>(N);
1502 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1503 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1504 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1505 canUseBlockOperation(StoreA, LoadB);
1506}
1507
1508bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1509
1510 auto *MemAccess = cast<LSBaseSDNode>(N);
1511 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1512 SDValue BasePtr = MemAccess->getBasePtr();
1513 MachineMemOperand *MMO = MemAccess->getMemOperand();
1514 assert(MMO && "Expected a memory operand.");
1515
1516 // The memory access must have a proper alignment and no index register.
1517 if (MemAccess->getAlign().value() < StoreSize ||
1518 !MemAccess->getOffset().isUndef())
1519 return false;
1520
1521 // The MMO must not have an unaligned offset.
1522 if (MMO->getOffset() % StoreSize != 0)
1523 return false;
1524
1525 // An access to GOT or the Constant Pool is aligned.
1526 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1527 if ((PSV->isGOT() || PSV->isConstantPool()))
1528 return true;
1529
1530 // Check the alignment of a Global Address.
1531 if (BasePtr.getNumOperands())
1532 if (GlobalAddressSDNode *GA =
1533 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1534 // The immediate offset must be aligned.
1535 if (GA->getOffset() % StoreSize != 0)
1536 return false;
1537
1538 // The alignment of the symbol itself must be at least the store size.
1539 const GlobalValue *GV = GA->getGlobal();
1540 const DataLayout &DL = GV->getParent()->getDataLayout();
1541 if (GV->getPointerAlignment(DL).value() < StoreSize)
1542 return false;
1543 }
1544
1545 return true;
1546}
1547
1548void SystemZDAGToDAGISel::Select(SDNode *Node) {
1549 // If we have a custom node, we already have selected!
1550 if (Node->isMachineOpcode()) {
1551 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1552 Node->setNodeId(-1);
1553 return;
1554 }
1555
1556 unsigned Opcode = Node->getOpcode();
1557 switch (Opcode) {
1558 case ISD::OR:
1559 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1560 if (tryRxSBG(Node, SystemZ::ROSBG))
1561 return;
1562 goto or_xor;
1563
1564 case ISD::XOR:
1565 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1566 if (tryRxSBG(Node, SystemZ::RXSBG))
1567 return;
1568 // Fall through.
1569 or_xor:
1570 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1571 // split the operation into two. If both operands here happen to be
1572 // constant, leave this to common code to optimize.
1573 if (Node->getValueType(0) == MVT::i64 &&
1574 Node->getOperand(0).getOpcode() != ISD::Constant)
1575 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1576 uint64_t Val = Op1->getZExtValue();
1577 // Don't split the operation if we can match one of the combined
1578 // logical operations provided by miscellaneous-extensions-3.
1579 if (Subtarget->hasMiscellaneousExtensions3()) {
1580 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1581 // Check whether this expression matches NAND/NOR/NXOR.
1582 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1583 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1584 ChildOpcode == ISD::XOR)
1585 break;
1586 // Check whether this expression matches OR-with-complement
1587 // (or matches an alternate pattern for NXOR).
1588 if (ChildOpcode == ISD::XOR) {
1589 auto Op0 = Node->getOperand(0);
1590 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1591 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1592 break;
1593 }
1594 }
1595 // Don't split an XOR with -1 as LCGR/AGHI is more compact.
1596 if (Opcode == ISD::XOR && Op1->isAllOnes())
1597 break;
1598 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1599 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1600 Val - uint32_t(Val), uint32_t(Val));
1601 return;
1602 }
1603 }
1604 break;
1605
1606 case ISD::AND:
1607 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1608 if (tryRxSBG(Node, SystemZ::RNSBG))
1609 return;
1610 [[fallthrough]];
1611 case ISD::ROTL:
1612 case ISD::SHL:
1613 case ISD::SRL:
1614 case ISD::ZERO_EXTEND:
1615 if (tryRISBGZero(Node))
1616 return;
1617 break;
1618
1619 case ISD::BSWAP:
1620 if (Node->getValueType(0) == MVT::i128) {
1621 SDLoc DL(Node);
1622 SDValue Src = Node->getOperand(0);
1623 Src = CurDAG->getNode(ISD::BITCAST, DL, MVT::v16i8, Src);
1624
1625 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1626 SDNode *Mask = loadPoolVectorConstant(APInt(128, Bytes), MVT::v16i8, DL);
1627 SDValue Ops[] = { Src, Src, SDValue(Mask, 0) };
1628 SDValue Res = SDValue(CurDAG->getMachineNode(SystemZ::VPERM, DL,
1629 MVT::v16i8, Ops), 0);
1630
1631 Res = CurDAG->getNode(ISD::BITCAST, DL, MVT::i128, Res);
1632 SDNode *ResNode = Res.getNode();
1633 ReplaceNode(Node, ResNode);
1634 SelectCode(Src.getNode());
1635 SelectCode(ResNode);
1636 return;
1637 }
1638 break;
1639
1640 case ISD::Constant:
1641 // If this is a 64-bit constant that is out of the range of LLILF,
1642 // LLIHF and LGFI, split it into two 32-bit pieces.
1643 if (Node->getValueType(0) == MVT::i64) {
1644 uint64_t Val = Node->getAsZExtVal();
1645 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1646 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1647 uint32_t(Val));
1648 return;
1649 }
1650 }
1651 if (Node->getValueType(0) == MVT::i128) {
1652 const APInt &Val = Node->getAsAPIntVal();
1654 if (VCI.isVectorConstantLegal(*Subtarget)) {
1655 loadVectorConstant(VCI, Node);
1656 return;
1657 }
1658 // If we can't materialize the constant we need to use a literal pool.
1659 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128, SDLoc(Node));
1660 ReplaceNode(Node, ResNode);
1661 return;
1662 }
1663 break;
1664
1666 SDValue Op0 = Node->getOperand(0);
1667 SDValue Op1 = Node->getOperand(1);
1668 // Prefer to put any load first, so that it can be matched as a
1669 // conditional load. Likewise for constants in range for LOCHI.
1670 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1671 (Subtarget->hasLoadStoreOnCond2() &&
1672 Node->getValueType(0).isInteger() &&
1673 Node->getValueType(0).getSizeInBits() <= 64 &&
1674 Op1.getOpcode() == ISD::Constant &&
1675 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1676 !(Op0.getOpcode() == ISD::Constant &&
1677 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1678 SDValue CCValid = Node->getOperand(2);
1679 SDValue CCMask = Node->getOperand(3);
1680 uint64_t ConstCCValid = CCValid.getNode()->getAsZExtVal();
1681 uint64_t ConstCCMask = CCMask.getNode()->getAsZExtVal();
1682 // Invert the condition.
1683 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1684 SDLoc(Node), CCMask.getValueType());
1685 SDValue Op4 = Node->getOperand(4);
1686 SDNode *UpdatedNode =
1687 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1688 if (UpdatedNode != Node) {
1689 // In case this node already exists then replace Node with it.
1690 ReplaceNode(Node, UpdatedNode);
1691 Node = UpdatedNode;
1692 }
1693 }
1694 break;
1695 }
1696
1698 EVT VT = Node->getValueType(0);
1699 unsigned ElemBitSize = VT.getScalarSizeInBits();
1700 if (ElemBitSize == 32) {
1701 if (tryGather(Node, SystemZ::VGEF))
1702 return;
1703 } else if (ElemBitSize == 64) {
1704 if (tryGather(Node, SystemZ::VGEG))
1705 return;
1706 }
1707 break;
1708 }
1709
1710 case ISD::BUILD_VECTOR: {
1711 auto *BVN = cast<BuildVectorSDNode>(Node);
1713 if (VCI.isVectorConstantLegal(*Subtarget)) {
1714 loadVectorConstant(VCI, Node);
1715 return;
1716 }
1717 break;
1718 }
1719
1720 case ISD::ConstantFP: {
1721 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1722 if (Imm.isZero() || Imm.isNegZero())
1723 break;
1725 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1726 assert(Success && "Expected legal FP immediate");
1727 loadVectorConstant(VCI, Node);
1728 return;
1729 }
1730
1731 case ISD::STORE: {
1732 if (tryFoldLoadStoreIntoMemOperand(Node))
1733 return;
1734 auto *Store = cast<StoreSDNode>(Node);
1735 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1736 if (ElemBitSize == 32) {
1737 if (tryScatter(Store, SystemZ::VSCEF))
1738 return;
1739 } else if (ElemBitSize == 64) {
1740 if (tryScatter(Store, SystemZ::VSCEG))
1741 return;
1742 }
1743 break;
1744 }
1745 }
1746
1747 SelectCode(Node);
1748}
1749
1750bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1751 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1752 std::vector<SDValue> &OutOps) {
1753 SystemZAddressingMode::AddrForm Form;
1754 SystemZAddressingMode::DispRange DispRange;
1755 SDValue Base, Disp, Index;
1756
1757 switch(ConstraintID) {
1758 default:
1759 llvm_unreachable("Unexpected asm memory constraint");
1760 case InlineAsm::ConstraintCode::i:
1761 case InlineAsm::ConstraintCode::Q:
1762 case InlineAsm::ConstraintCode::ZQ:
1763 // Accept an address with a short displacement, but no index.
1764 Form = SystemZAddressingMode::FormBD;
1765 DispRange = SystemZAddressingMode::Disp12Only;
1766 break;
1767 case InlineAsm::ConstraintCode::R:
1768 case InlineAsm::ConstraintCode::ZR:
1769 // Accept an address with a short displacement and an index.
1770 Form = SystemZAddressingMode::FormBDXNormal;
1771 DispRange = SystemZAddressingMode::Disp12Only;
1772 break;
1773 case InlineAsm::ConstraintCode::S:
1774 case InlineAsm::ConstraintCode::ZS:
1775 // Accept an address with a long displacement, but no index.
1776 Form = SystemZAddressingMode::FormBD;
1777 DispRange = SystemZAddressingMode::Disp20Only;
1778 break;
1779 case InlineAsm::ConstraintCode::T:
1780 case InlineAsm::ConstraintCode::m:
1781 case InlineAsm::ConstraintCode::o:
1782 case InlineAsm::ConstraintCode::p:
1783 case InlineAsm::ConstraintCode::ZT:
1784 // Accept an address with a long displacement and an index.
1785 // m works the same as T, as this is the most general case.
1786 // We don't really have any special handling of "offsettable"
1787 // memory addresses, so just treat o the same as m.
1788 Form = SystemZAddressingMode::FormBDXNormal;
1789 DispRange = SystemZAddressingMode::Disp20Only;
1790 break;
1791 }
1792
1793 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1794 const TargetRegisterClass *TRC =
1795 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1796 SDLoc DL(Base);
1797 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1798
1799 // Make sure that the base address doesn't go into %r0.
1800 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1801 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1802 Base.getOpcode() != ISD::Register) {
1803 Base =
1804 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1805 DL, Base.getValueType(),
1806 Base, RC), 0);
1807 }
1808
1809 // Make sure that the index register isn't assigned to %r0 either.
1810 if (Index.getOpcode() != ISD::Register) {
1811 Index =
1812 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1813 DL, Index.getValueType(),
1814 Index, RC), 0);
1815 }
1816
1817 OutOps.push_back(Base);
1818 OutOps.push_back(Disp);
1819 OutOps.push_back(Index);
1820 return false;
1821 }
1822
1823 return true;
1824}
1825
1826// IsProfitableToFold - Returns true if is profitable to fold the specific
1827// operand node N of U during instruction selection that starts at Root.
1828bool
1829SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1830 SDNode *Root) const {
1831 // We want to avoid folding a LOAD into an ICMP node if as a result
1832 // we would be forced to spill the condition code into a GPR.
1833 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1834 if (!N.hasOneUse() || !U->hasOneUse())
1835 return false;
1836
1837 // The user of the CC value will usually be a CopyToReg into the
1838 // physical CC register, which in turn is glued and chained to the
1839 // actual instruction that uses the CC value. Bail out if we have
1840 // anything else than that.
1841 SDNode *CCUser = *U->use_begin();
1842 SDNode *CCRegUser = nullptr;
1843 if (CCUser->getOpcode() == ISD::CopyToReg ||
1844 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1845 for (auto *U : CCUser->uses()) {
1846 if (CCRegUser == nullptr)
1847 CCRegUser = U;
1848 else if (CCRegUser != U)
1849 return false;
1850 }
1851 }
1852 if (CCRegUser == nullptr)
1853 return false;
1854
1855 // If the actual instruction is a branch, the only thing that remains to be
1856 // checked is whether the CCUser chain is a predecessor of the load.
1857 if (CCRegUser->isMachineOpcode() &&
1858 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1859 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1860
1861 // Otherwise, the instruction may have multiple operands, and we need to
1862 // verify that none of them are a predecessor of the load. This is exactly
1863 // the same check that would be done by common code if the CC setter were
1864 // glued to the CC user, so simply invoke that check here.
1865 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1866 return false;
1867 }
1868
1869 return true;
1870}
1871
1872namespace {
1873// Represents a sequence for extracting a 0/1 value from an IPM result:
1874// (((X ^ XORValue) + AddValue) >> Bit)
1875struct IPMConversion {
1876 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1877 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1878
1879 int64_t XORValue;
1880 int64_t AddValue;
1881 unsigned Bit;
1882};
1883} // end anonymous namespace
1884
1885// Return a sequence for getting a 1 from an IPM result when CC has a
1886// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1887// The handling of CC values outside CCValid doesn't matter.
1888static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1889 // Deal with cases where the result can be taken directly from a bit
1890 // of the IPM result.
1891 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1892 return IPMConversion(0, 0, SystemZ::IPM_CC);
1893 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1894 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1895
1896 // Deal with cases where we can add a value to force the sign bit
1897 // to contain the right value. Putting the bit in 31 means we can
1898 // use SRL rather than RISBG(L), and also makes it easier to get a
1899 // 0/-1 value, so it has priority over the other tests below.
1900 //
1901 // These sequences rely on the fact that the upper two bits of the
1902 // IPM result are zero.
1903 uint64_t TopBit = uint64_t(1) << 31;
1904 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1905 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1906 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1907 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1908 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1911 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1912 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1913 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1914 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1917 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1918
1919 // Next try inverting the value and testing a bit. 0/1 could be
1920 // handled this way too, but we dealt with that case above.
1921 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1922 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1923
1924 // Handle cases where adding a value forces a non-sign bit to contain
1925 // the right value.
1926 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1927 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1928 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1929 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1930
1931 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1932 // can be done by inverting the low CC bit and applying one of the
1933 // sign-based extractions above.
1934 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1935 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1936 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1937 return IPMConversion(1 << SystemZ::IPM_CC,
1938 TopBit - (3 << SystemZ::IPM_CC), 31);
1939 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1942 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1943 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1946 return IPMConversion(1 << SystemZ::IPM_CC,
1947 TopBit - (1 << SystemZ::IPM_CC), 31);
1948
1949 llvm_unreachable("Unexpected CC combination");
1950}
1951
1952SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1953 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1954 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1955 if (!TrueOp || !FalseOp)
1956 return SDValue();
1957 if (FalseOp->getZExtValue() != 0)
1958 return SDValue();
1959 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1960 return SDValue();
1961
1962 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1963 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1964 if (!CCValidOp || !CCMaskOp)
1965 return SDValue();
1966 int CCValid = CCValidOp->getZExtValue();
1967 int CCMask = CCMaskOp->getZExtValue();
1968
1969 SDLoc DL(Node);
1970 SDValue CCReg = Node->getOperand(4);
1971 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1972 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1973
1974 if (IPM.XORValue)
1975 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1976 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1977
1978 if (IPM.AddValue)
1979 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1980 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1981
1982 EVT VT = Node->getValueType(0);
1983 if (VT == MVT::i32 && IPM.Bit == 31) {
1984 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1985 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1986 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1987 } else {
1988 if (VT != MVT::i32)
1989 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1990
1991 if (TrueOp->getSExtValue() == 1) {
1992 // The SHR/AND sequence should get optimized to an RISBG.
1993 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1994 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1995 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1996 CurDAG->getConstant(1, DL, VT));
1997 } else {
1998 // Sign-extend from IPM.Bit using a pair of shifts.
1999 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
2000 int SraAmt = VT.getSizeInBits() - 1;
2001 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
2002 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
2003 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
2004 CurDAG->getConstant(SraAmt, DL, MVT::i32));
2005 }
2006 }
2007
2008 return Result;
2009}
2010
2011void SystemZDAGToDAGISel::PreprocessISelDAG() {
2012 // If we have conditional immediate loads, we always prefer
2013 // using those over an IPM sequence.
2014 if (Subtarget->hasLoadStoreOnCond2())
2015 return;
2016
2017 bool MadeChange = false;
2018
2019 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
2020 E = CurDAG->allnodes_end();
2021 I != E;) {
2022 SDNode *N = &*I++;
2023 if (N->use_empty())
2024 continue;
2025
2026 SDValue Res;
2027 switch (N->getOpcode()) {
2028 default: break;
2030 Res = expandSelectBoolean(N);
2031 break;
2032 }
2033
2034 if (Res) {
2035 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
2036 LLVM_DEBUG(N->dump(CurDAG));
2037 LLVM_DEBUG(dbgs() << "\nNew: ");
2038 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
2039 LLVM_DEBUG(dbgs() << "\n");
2040
2041 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
2042 MadeChange = true;
2043 }
2044 }
2045
2046 if (MadeChange)
2047 CurDAG->RemoveDeadNodes();
2048}
#define Success
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
const HexagonInstrInfo * TII
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
#define PASS_NAME
#define DEBUG_TYPE
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
Definition: APInt.h:76
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1491
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
@ MOLoad
The memory access reads data.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:287
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:534
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static IntegerType * getInt128Ty(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition: Value.cpp:926
self_iterator getIterator()
Definition: ilist_node.h:109
#define UINT64_MAX
Definition: DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
const unsigned CCMASK_0
Definition: SystemZ.h:27
const unsigned CCMASK_1
Definition: SystemZ.h:28
const unsigned IPM_CC
Definition: SystemZ.h:112
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:186
const unsigned CCMASK_3
Definition: SystemZ.h:30
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:181
const unsigned CCMASK_2
Definition: SystemZ.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition: DWP.cpp:456
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)