LLVM 19.0.0git
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the SystemZ target.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SystemZISelLowering.h"
17#include "llvm/Support/Debug.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
25
26namespace {
27// Used to build addressing modes.
28struct SystemZAddressingMode {
29 // The shape of the address.
30 enum AddrForm {
31 // base+displacement
32 FormBD,
33
34 // base+displacement+index for load and store operands
35 FormBDXNormal,
36
37 // base+displacement+index for load address operands
38 FormBDXLA,
39
40 // base+displacement+index+ADJDYNALLOC
41 FormBDXDynAlloc
42 };
43 AddrForm Form;
44
45 // The type of displacement. The enum names here correspond directly
46 // to the definitions in SystemZOperand.td. We could split them into
47 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
48 enum DispRange {
49 Disp12Only,
50 Disp12Pair,
51 Disp20Only,
52 Disp20Only128,
53 Disp20Pair
54 };
55 DispRange DR;
56
57 // The parts of the address. The address is equivalent to:
58 //
59 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
61 int64_t Disp;
63 bool IncludesDynAlloc;
64
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
67
68 // True if the address can have an index register.
69 bool hasIndexField() { return Form != FormBD; }
70
71 // True if the address can (and must) include ADJDYNALLOC.
72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
76
77 errs() << " Base ";
78 if (Base.getNode())
79 Base.getNode()->dump(DAG);
80 else
81 errs() << "null\n";
82
83 if (hasIndexField()) {
84 errs() << " Index ";
85 if (Index.getNode())
86 Index.getNode()->dump(DAG);
87 else
88 errs() << "null\n";
89 }
90
91 errs() << " Disp " << Disp;
92 if (IncludesDynAlloc)
93 errs() << " + ADJDYNALLOC";
94 errs() << '\n';
95 }
96};
97
98// Return a mask with Count low bits set.
99static uint64_t allOnes(unsigned int Count) {
100 assert(Count <= 64);
101 if (Count > 63)
102 return UINT64_MAX;
103 return (uint64_t(1) << Count) - 1;
104}
105
106// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108// Rotate (I5). The combined operand value is effectively:
109//
110// (or (rotl Input, Rotate), ~Mask)
111//
112// for RNSBG and:
113//
114// (and (rotl Input, Rotate), Mask)
115//
116// otherwise. The output value has BitSize bits, although Input may be
117// narrower (in which case the upper bits are don't care), or wider (in which
118// case the result will be truncated as part of the operation).
119struct RxSBGOperands {
120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123 Rotate(0) {}
124
125 unsigned Opcode;
126 unsigned BitSize;
127 uint64_t Mask;
128 SDValue Input;
129 unsigned Start;
130 unsigned End;
131 unsigned Rotate;
132};
133
134class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
136
137 // Used by SystemZOperands.td to create integer constants.
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140 }
141
142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
144 }
145
146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
148 }
149
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
168
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
174
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182 // PC-relative address matching routines used by SystemZOperands.td.
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
186 return true;
187 }
188 return false;
189 }
190
191 // BD matching routines used by SystemZOperands.td.
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194 }
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197 }
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200 }
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203 }
204
205 // MVI matching routines used by SystemZOperands.td.
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208 }
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211 }
212
213 // BDX matching routines used by SystemZOperands.td.
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
219 }
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
225 }
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
231 }
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
237 }
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
243 }
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
249 }
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
255 }
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
261 }
262
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
269
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
272 // set Op to that Y.
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
292
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297 // If Op0 is null, then Node is a constant that can be loaded using:
298 //
299 // (Opcode UpperVal LowerVal)
300 //
301 // If Op0 is nonnull, then Node can be implemented using:
302 //
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
306
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308 SDNode *Node);
309
310 SDNode *loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL);
311
312 // Try to use gather instruction Opcode to implement vector insertion N.
313 bool tryGather(SDNode *N, unsigned Opcode);
314
315 // Try to use scatter instruction Opcode to implement store Store.
316 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
317
318 // Change a chain of {load; op; store} of the same value into a simple op
319 // through memory of that value, if the uses of the modified value and its
320 // address are suitable.
321 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
322
323 // Return true if Load and Store are loads and stores of the same size
324 // and are guaranteed not to overlap. Such operations can be implemented
325 // using block (SS-format) instructions.
326 //
327 // Partial overlap would lead to incorrect code, since the block operations
328 // are logically bytewise, even though they have a fast path for the
329 // non-overlapping case. We also need to avoid full overlap (i.e. two
330 // addresses that might be equal at run time) because although that case
331 // would be handled correctly, it might be implemented by millicode.
332 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
333
334 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
335 // from Y to X.
336 bool storeLoadCanUseMVC(SDNode *N) const;
337
338 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
339 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
340 // to X.
341 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
342
343 // Return true if N (a load or a store) fullfills the alignment
344 // requirements for a PC-relative access.
345 bool storeLoadIsAligned(SDNode *N) const;
346
347 // Return the load extension type of a load or atomic load.
348 ISD::LoadExtType getLoadExtType(SDNode *N) const;
349
350 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
351 SDValue expandSelectBoolean(SDNode *Node);
352
353 // Return true if the flags of N and the subtarget allows for
354 // reassociation, in which case a reg/reg opcode is needed as input to the
355 // MachineCombiner.
356 bool shouldSelectForReassoc(SDNode *N) const;
357
358public:
359 static char ID;
360
361 SystemZDAGToDAGISel() = delete;
362
363 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
364 : SelectionDAGISel(ID, TM, OptLevel) {}
365
366 bool runOnMachineFunction(MachineFunction &MF) override {
367 const Function &F = MF.getFunction();
368 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
369 if (F.hasFnAttribute("mnop-mcount"))
370 report_fatal_error("mnop-mcount only supported with fentry-call");
371 if (F.hasFnAttribute("mrecord-mcount"))
372 report_fatal_error("mrecord-mcount only supported with fentry-call");
373 }
374
375 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
377 }
378
379 // Override SelectionDAGISel.
380 void Select(SDNode *Node) override;
382 InlineAsm::ConstraintCode ConstraintID,
383 std::vector<SDValue> &OutOps) override;
384 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
385 void PreprocessISelDAG() override;
386
387 // Include the pieces autogenerated from the target description.
388 #include "SystemZGenDAGISel.inc"
389};
390} // end anonymous namespace
391
392char SystemZDAGToDAGISel::ID = 0;
393
394INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)
395
397 CodeGenOptLevel OptLevel) {
398 return new SystemZDAGToDAGISel(TM, OptLevel);
399}
400
401// Return true if Val should be selected as a displacement for an address
402// with range DR. Here we're interested in the range of both the instruction
403// described by DR and of any pairing instruction.
404static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
405 switch (DR) {
406 case SystemZAddressingMode::Disp12Only:
407 return isUInt<12>(Val);
408
409 case SystemZAddressingMode::Disp12Pair:
410 case SystemZAddressingMode::Disp20Only:
411 case SystemZAddressingMode::Disp20Pair:
412 return isInt<20>(Val);
413
414 case SystemZAddressingMode::Disp20Only128:
415 return isInt<20>(Val) && isInt<20>(Val + 8);
416 }
417 llvm_unreachable("Unhandled displacement range");
418}
419
420// Change the base or index in AM to Value, where IsBase selects
421// between the base and index.
422static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
423 SDValue Value) {
424 if (IsBase)
425 AM.Base = Value;
426 else
427 AM.Index = Value;
428}
429
430// The base or index of AM is equivalent to Value + ADJDYNALLOC,
431// where IsBase selects between the base and index. Try to fold the
432// ADJDYNALLOC into AM.
433static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
434 SDValue Value) {
435 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
436 changeComponent(AM, IsBase, Value);
437 AM.IncludesDynAlloc = true;
438 return true;
439 }
440 return false;
441}
442
443// The base of AM is equivalent to Base + Index. Try to use Index as
444// the index register.
445static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
446 SDValue Index) {
447 if (AM.hasIndexField() && !AM.Index.getNode()) {
448 AM.Base = Base;
449 AM.Index = Index;
450 return true;
451 }
452 return false;
453}
454
455// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
456// between the base and index. Try to fold Op1 into AM's displacement.
457static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
458 SDValue Op0, uint64_t Op1) {
459 // First try adjusting the displacement.
460 int64_t TestDisp = AM.Disp + Op1;
461 if (selectDisp(AM.DR, TestDisp)) {
462 changeComponent(AM, IsBase, Op0);
463 AM.Disp = TestDisp;
464 return true;
465 }
466
467 // We could consider forcing the displacement into a register and
468 // using it as an index, but it would need to be carefully tuned.
469 return false;
470}
471
472bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
473 bool IsBase) const {
474 SDValue N = IsBase ? AM.Base : AM.Index;
475 unsigned Opcode = N.getOpcode();
476 // Look through no-op truncations.
477 if (Opcode == ISD::TRUNCATE && N.getOperand(0).getValueSizeInBits() <= 64) {
478 N = N.getOperand(0);
479 Opcode = N.getOpcode();
480 }
481 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
482 SDValue Op0 = N.getOperand(0);
483 SDValue Op1 = N.getOperand(1);
484
485 unsigned Op0Code = Op0->getOpcode();
486 unsigned Op1Code = Op1->getOpcode();
487
488 if (Op0Code == SystemZISD::ADJDYNALLOC)
489 return expandAdjDynAlloc(AM, IsBase, Op1);
490 if (Op1Code == SystemZISD::ADJDYNALLOC)
491 return expandAdjDynAlloc(AM, IsBase, Op0);
492
493 if (Op0Code == ISD::Constant)
494 return expandDisp(AM, IsBase, Op1,
495 cast<ConstantSDNode>(Op0)->getSExtValue());
496 if (Op1Code == ISD::Constant)
497 return expandDisp(AM, IsBase, Op0,
498 cast<ConstantSDNode>(Op1)->getSExtValue());
499
500 if (IsBase && expandIndex(AM, Op0, Op1))
501 return true;
502 }
503 if (Opcode == SystemZISD::PCREL_OFFSET) {
504 SDValue Full = N.getOperand(0);
505 SDValue Base = N.getOperand(1);
506 SDValue Anchor = Base.getOperand(0);
507 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
508 cast<GlobalAddressSDNode>(Anchor)->getOffset());
509 return expandDisp(AM, IsBase, Base, Offset);
510 }
511 return false;
512}
513
514// Return true if an instruction with displacement range DR should be
515// used for displacement value Val. selectDisp(DR, Val) must already hold.
516static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
517 assert(selectDisp(DR, Val) && "Invalid displacement");
518 switch (DR) {
519 case SystemZAddressingMode::Disp12Only:
520 case SystemZAddressingMode::Disp20Only:
521 case SystemZAddressingMode::Disp20Only128:
522 return true;
523
524 case SystemZAddressingMode::Disp12Pair:
525 // Use the other instruction if the displacement is too large.
526 return isUInt<12>(Val);
527
528 case SystemZAddressingMode::Disp20Pair:
529 // Use the other instruction if the displacement is small enough.
530 return !isUInt<12>(Val);
531 }
532 llvm_unreachable("Unhandled displacement range");
533}
534
535// Return true if Base + Disp + Index should be performed by LA(Y).
536static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
537 // Don't use LA(Y) for constants.
538 if (!Base)
539 return false;
540
541 // Always use LA(Y) for frame addresses, since we know that the destination
542 // register is almost always (perhaps always) going to be different from
543 // the frame register.
544 if (Base->getOpcode() == ISD::FrameIndex)
545 return true;
546
547 if (Disp) {
548 // Always use LA(Y) if there is a base, displacement and index.
549 if (Index)
550 return true;
551
552 // Always use LA if the displacement is small enough. It should always
553 // be no worse than AGHI (and better if it avoids a move).
554 if (isUInt<12>(Disp))
555 return true;
556
557 // For similar reasons, always use LAY if the constant is too big for AGHI.
558 // LAY should be no worse than AGFI.
559 if (!isInt<16>(Disp))
560 return true;
561 } else {
562 // Don't use LA for plain registers.
563 if (!Index)
564 return false;
565
566 // Don't use LA for plain addition if the index operand is only used
567 // once. It should be a natural two-operand addition in that case.
568 if (Index->hasOneUse())
569 return false;
570
571 // Prefer addition if the second operation is sign-extended, in the
572 // hope of using AGF.
573 unsigned IndexOpcode = Index->getOpcode();
574 if (IndexOpcode == ISD::SIGN_EXTEND ||
575 IndexOpcode == ISD::SIGN_EXTEND_INREG)
576 return false;
577 }
578
579 // Don't use LA for two-operand addition if either operand is only
580 // used once. The addition instructions are better in that case.
581 if (Base->hasOneUse())
582 return false;
583
584 return true;
585}
586
587// Return true if Addr is suitable for AM, updating AM if so.
588bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
589 SystemZAddressingMode &AM) const {
590 // Start out assuming that the address will need to be loaded separately,
591 // then try to extend it as much as we can.
592 AM.Base = Addr;
593
594 // First try treating the address as a constant.
595 if (Addr.getOpcode() == ISD::Constant &&
596 expandDisp(AM, true, SDValue(),
597 cast<ConstantSDNode>(Addr)->getSExtValue()))
598 ;
599 // Also see if it's a bare ADJDYNALLOC.
600 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
601 expandAdjDynAlloc(AM, true, SDValue()))
602 ;
603 else
604 // Otherwise try expanding each component.
605 while (expandAddress(AM, true) ||
606 (AM.Index.getNode() && expandAddress(AM, false)))
607 continue;
608
609 // Reject cases where it isn't profitable to use LA(Y).
610 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
611 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
612 return false;
613
614 // Reject cases where the other instruction in a pair should be used.
615 if (!isValidDisp(AM.DR, AM.Disp))
616 return false;
617
618 // Make sure that ADJDYNALLOC is included where necessary.
619 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
620 return false;
621
622 LLVM_DEBUG(AM.dump(CurDAG));
623 return true;
624}
625
626// Insert a node into the DAG at least before Pos. This will reposition
627// the node as needed, and will assign it a node ID that is <= Pos's ID.
628// Note that this does *not* preserve the uniqueness of node IDs!
629// The selection DAG must no longer depend on their uniqueness when this
630// function is used.
631static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
632 if (N->getNodeId() == -1 ||
635 DAG->RepositionNode(Pos->getIterator(), N.getNode());
636 // Mark Node as invalid for pruning as after this it may be a successor to a
637 // selected node but otherwise be in the same position of Pos.
638 // Conservatively mark it with the same -abs(Id) to assure node id
639 // invariant is preserved.
640 N->setNodeId(Pos->getNodeId());
642 }
643}
644
645void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
646 EVT VT, SDValue &Base,
647 SDValue &Disp) const {
648 Base = AM.Base;
649 if (!Base.getNode())
650 // Register 0 means "no base". This is mostly useful for shifts.
651 Base = CurDAG->getRegister(0, VT);
652 else if (Base.getOpcode() == ISD::FrameIndex) {
653 // Lower a FrameIndex to a TargetFrameIndex.
654 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
655 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
656 } else if (Base.getValueType() != VT) {
657 // Truncate values from i64 to i32, for shifts.
658 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
659 "Unexpected truncation");
660 SDLoc DL(Base);
661 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
662 insertDAGNode(CurDAG, Base.getNode(), Trunc);
663 Base = Trunc;
664 }
665
666 // Lower the displacement to a TargetConstant.
667 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
668}
669
670void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
671 EVT VT, SDValue &Base,
672 SDValue &Disp,
673 SDValue &Index) const {
674 getAddressOperands(AM, VT, Base, Disp);
675
676 Index = AM.Index;
677 if (!Index.getNode())
678 // Register 0 means "no index".
679 Index = CurDAG->getRegister(0, VT);
680}
681
682bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
684 SDValue &Disp) const {
685 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
686 if (!selectAddress(Addr, AM))
687 return false;
688
689 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
690 return true;
691}
692
693bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
695 SDValue &Disp) const {
696 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
697 if (!selectAddress(Addr, AM) || AM.Index.getNode())
698 return false;
699
700 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
701 return true;
702}
703
704bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
705 SystemZAddressingMode::DispRange DR,
707 SDValue &Disp, SDValue &Index) const {
708 SystemZAddressingMode AM(Form, DR);
709 if (!selectAddress(Addr, AM))
710 return false;
711
712 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
713 return true;
714}
715
716bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
717 SDValue &Base,
718 SDValue &Disp,
719 SDValue &Index) const {
720 SDValue Regs[2];
721 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
722 Regs[0].getNode() && Regs[1].getNode()) {
723 for (unsigned int I = 0; I < 2; ++I) {
724 Base = Regs[I];
725 Index = Regs[1 - I];
726 // We can't tell here whether the index vector has the right type
727 // for the access; the caller needs to do that instead.
728 if (Index.getOpcode() == ISD::ZERO_EXTEND)
730 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
731 Index.getOperand(1) == Elem) {
733 return true;
734 }
735 }
736 }
737 return false;
738}
739
740bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
741 uint64_t InsertMask) const {
742 // We're only interested in cases where the insertion is into some operand
743 // of Op, rather than into Op itself. The only useful case is an AND.
744 if (Op.getOpcode() != ISD::AND)
745 return false;
746
747 // We need a constant mask.
748 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
749 if (!MaskNode)
750 return false;
751
752 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
753 uint64_t AndMask = MaskNode->getZExtValue();
754 if (InsertMask & AndMask)
755 return false;
756
757 // It's only an insertion if all bits are covered or are known to be zero.
758 // The inner check covers all cases but is more expensive.
759 uint64_t Used = allOnes(Op.getValueSizeInBits());
760 if (Used != (AndMask | InsertMask)) {
761 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
762 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
763 return false;
764 }
765
766 Op = Op.getOperand(0);
767 return true;
768}
769
770bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
771 uint64_t Mask) const {
772 const SystemZInstrInfo *TII = getInstrInfo();
773 if (RxSBG.Rotate != 0)
774 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
775 Mask &= RxSBG.Mask;
776 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
777 RxSBG.Mask = Mask;
778 return true;
779 }
780 return false;
781}
782
783// Return true if any bits of (RxSBG.Input & Mask) are significant.
784static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
785 // Rotate the mask in the same way as RxSBG.Input is rotated.
786 if (RxSBG.Rotate != 0)
787 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
788 return (Mask & RxSBG.Mask) != 0;
789}
790
791bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
792 SDValue N = RxSBG.Input;
793 unsigned Opcode = N.getOpcode();
794 switch (Opcode) {
795 case ISD::TRUNCATE: {
796 if (RxSBG.Opcode == SystemZ::RNSBG)
797 return false;
798 if (N.getOperand(0).getValueSizeInBits() > 64)
799 return false;
800 uint64_t BitSize = N.getValueSizeInBits();
801 uint64_t Mask = allOnes(BitSize);
802 if (!refineRxSBGMask(RxSBG, Mask))
803 return false;
804 RxSBG.Input = N.getOperand(0);
805 return true;
806 }
807 case ISD::AND: {
808 if (RxSBG.Opcode == SystemZ::RNSBG)
809 return false;
810
811 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
812 if (!MaskNode)
813 return false;
814
815 SDValue Input = N.getOperand(0);
816 uint64_t Mask = MaskNode->getZExtValue();
817 if (!refineRxSBGMask(RxSBG, Mask)) {
818 // If some bits of Input are already known zeros, those bits will have
819 // been removed from the mask. See if adding them back in makes the
820 // mask suitable.
821 KnownBits Known = CurDAG->computeKnownBits(Input);
822 Mask |= Known.Zero.getZExtValue();
823 if (!refineRxSBGMask(RxSBG, Mask))
824 return false;
825 }
826 RxSBG.Input = Input;
827 return true;
828 }
829
830 case ISD::OR: {
831 if (RxSBG.Opcode != SystemZ::RNSBG)
832 return false;
833
834 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
835 if (!MaskNode)
836 return false;
837
838 SDValue Input = N.getOperand(0);
839 uint64_t Mask = ~MaskNode->getZExtValue();
840 if (!refineRxSBGMask(RxSBG, Mask)) {
841 // If some bits of Input are already known ones, those bits will have
842 // been removed from the mask. See if adding them back in makes the
843 // mask suitable.
844 KnownBits Known = CurDAG->computeKnownBits(Input);
845 Mask &= ~Known.One.getZExtValue();
846 if (!refineRxSBGMask(RxSBG, Mask))
847 return false;
848 }
849 RxSBG.Input = Input;
850 return true;
851 }
852
853 case ISD::ROTL: {
854 // Any 64-bit rotate left can be merged into the RxSBG.
855 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
856 return false;
857 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
858 if (!CountNode)
859 return false;
860
861 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
862 RxSBG.Input = N.getOperand(0);
863 return true;
864 }
865
866 case ISD::ANY_EXTEND:
867 // Bits above the extended operand are don't-care.
868 RxSBG.Input = N.getOperand(0);
869 return true;
870
871 case ISD::ZERO_EXTEND:
872 if (RxSBG.Opcode != SystemZ::RNSBG) {
873 // Restrict the mask to the extended operand.
874 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
875 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
876 return false;
877
878 RxSBG.Input = N.getOperand(0);
879 return true;
880 }
881 [[fallthrough]];
882
883 case ISD::SIGN_EXTEND: {
884 // Check that the extension bits are don't-care (i.e. are masked out
885 // by the final mask).
886 unsigned BitSize = N.getValueSizeInBits();
887 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
888 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
889 // In the case where only the sign bit is active, increase Rotate with
890 // the extension width.
891 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
892 RxSBG.Rotate += (BitSize - InnerBitSize);
893 else
894 return false;
895 }
896
897 RxSBG.Input = N.getOperand(0);
898 return true;
899 }
900
901 case ISD::SHL: {
902 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
903 if (!CountNode)
904 return false;
905
906 uint64_t Count = CountNode->getZExtValue();
907 unsigned BitSize = N.getValueSizeInBits();
908 if (Count < 1 || Count >= BitSize)
909 return false;
910
911 if (RxSBG.Opcode == SystemZ::RNSBG) {
912 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
913 // count bits from RxSBG.Input are ignored.
914 if (maskMatters(RxSBG, allOnes(Count)))
915 return false;
916 } else {
917 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
918 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
919 return false;
920 }
921
922 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
923 RxSBG.Input = N.getOperand(0);
924 return true;
925 }
926
927 case ISD::SRL:
928 case ISD::SRA: {
929 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
930 if (!CountNode)
931 return false;
932
933 uint64_t Count = CountNode->getZExtValue();
934 unsigned BitSize = N.getValueSizeInBits();
935 if (Count < 1 || Count >= BitSize)
936 return false;
937
938 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
939 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
940 // count bits from RxSBG.Input are ignored.
941 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
942 return false;
943 } else {
944 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
945 // which is similar to SLL above.
946 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
947 return false;
948 }
949
950 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
951 RxSBG.Input = N.getOperand(0);
952 return true;
953 }
954 default:
955 return false;
956 }
957}
958
959SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
960 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
961 return SDValue(N, 0);
962}
963
964SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
965 SDValue N) const {
966 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
967 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
968 DL, VT, getUNDEF(DL, MVT::i64), N);
969 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
970 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
971 assert(N.getValueType() == VT && "Unexpected value types");
972 return N;
973}
974
975bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
976 SDLoc DL(N);
977 EVT VT = N->getValueType(0);
978 if (!VT.isInteger() || VT.getSizeInBits() > 64)
979 return false;
980 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
981 unsigned Count = 0;
982 while (expandRxSBG(RISBG))
983 // The widening or narrowing is expected to be free.
984 // Counting widening or narrowing as a saved operation will result in
985 // preferring an R*SBG over a simple shift/logical instruction.
986 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
987 RISBG.Input.getOpcode() != ISD::TRUNCATE)
988 Count += 1;
989 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
990 return false;
991
992 // Prefer to use normal shift instructions over RISBG, since they can handle
993 // all cases and are sometimes shorter.
994 if (Count == 1 && N->getOpcode() != ISD::AND)
995 return false;
996
997 // Prefer register extensions like LLC over RISBG. Also prefer to start
998 // out with normal ANDs if one instruction would be enough. We can convert
999 // these ANDs into an RISBG later if a three-address instruction is useful.
1000 if (RISBG.Rotate == 0) {
1001 bool PreferAnd = false;
1002 // Prefer AND for any 32-bit and-immediate operation.
1003 if (VT == MVT::i32)
1004 PreferAnd = true;
1005 // As well as for any 64-bit operation that can be implemented via LLC(R),
1006 // LLH(R), LLGT(R), or one of the and-immediate instructions.
1007 else if (RISBG.Mask == 0xff ||
1008 RISBG.Mask == 0xffff ||
1009 RISBG.Mask == 0x7fffffff ||
1010 SystemZ::isImmLF(~RISBG.Mask) ||
1011 SystemZ::isImmHF(~RISBG.Mask))
1012 PreferAnd = true;
1013 // And likewise for the LLZRGF instruction, which doesn't have a register
1014 // to register version.
1015 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1016 if (Load->getMemoryVT() == MVT::i32 &&
1017 (Load->getExtensionType() == ISD::EXTLOAD ||
1018 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1019 RISBG.Mask == 0xffffff00 &&
1020 Subtarget->hasLoadAndZeroRightmostByte())
1021 PreferAnd = true;
1022 }
1023 if (PreferAnd) {
1024 // Replace the current node with an AND. Note that the current node
1025 // might already be that same AND, in which case it is already CSE'd
1026 // with it, and we must not call ReplaceNode.
1027 SDValue In = convertTo(DL, VT, RISBG.Input);
1028 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1029 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1030 if (N != New.getNode()) {
1031 insertDAGNode(CurDAG, N, Mask);
1032 insertDAGNode(CurDAG, N, New);
1033 ReplaceNode(N, New.getNode());
1034 N = New.getNode();
1035 }
1036 // Now, select the machine opcode to implement this operation.
1037 if (!N->isMachineOpcode())
1038 SelectCode(N);
1039 return true;
1040 }
1041 }
1042
1043 unsigned Opcode = SystemZ::RISBG;
1044 // Prefer RISBGN if available, since it does not clobber CC.
1045 if (Subtarget->hasMiscellaneousExtensions())
1046 Opcode = SystemZ::RISBGN;
1047 EVT OpcodeVT = MVT::i64;
1048 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1049 // We can only use the 32-bit instructions if all source bits are
1050 // in the low 32 bits without wrapping, both after rotation (because
1051 // of the smaller range for Start and End) and before rotation
1052 // (because the input value is truncated).
1053 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1054 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1055 ((RISBG.End + RISBG.Rotate) & 63) >=
1056 ((RISBG.Start + RISBG.Rotate) & 63)) {
1057 Opcode = SystemZ::RISBMux;
1058 OpcodeVT = MVT::i32;
1059 RISBG.Start &= 31;
1060 RISBG.End &= 31;
1061 }
1062 SDValue Ops[5] = {
1063 getUNDEF(DL, OpcodeVT),
1064 convertTo(DL, OpcodeVT, RISBG.Input),
1065 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1066 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1067 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1068 };
1069 SDValue New = convertTo(
1070 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1071 ReplaceNode(N, New.getNode());
1072 return true;
1073}
1074
1075bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1076 SDLoc DL(N);
1077 EVT VT = N->getValueType(0);
1078 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1079 return false;
1080 // Try treating each operand of N as the second operand of the RxSBG
1081 // and see which goes deepest.
1082 RxSBGOperands RxSBG[] = {
1083 RxSBGOperands(Opcode, N->getOperand(0)),
1084 RxSBGOperands(Opcode, N->getOperand(1))
1085 };
1086 unsigned Count[] = { 0, 0 };
1087 for (unsigned I = 0; I < 2; ++I)
1088 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I]))
1089 // In cases of multiple users it seems better to keep the simple
1090 // instruction as they are one cycle faster, and it also helps in cases
1091 // where both inputs share a common node.
1092 // The widening or narrowing is expected to be free. Counting widening
1093 // or narrowing as a saved operation will result in preferring an R*SBG
1094 // over a simple shift/logical instruction.
1095 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1096 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1097 Count[I] += 1;
1098
1099 // Do nothing if neither operand is suitable.
1100 if (Count[0] == 0 && Count[1] == 0)
1101 return false;
1102
1103 // Pick the deepest second operand.
1104 unsigned I = Count[0] > Count[1] ? 0 : 1;
1105 SDValue Op0 = N->getOperand(I ^ 1);
1106
1107 // Prefer IC for character insertions from memory.
1108 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1109 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1110 if (Load->getMemoryVT() == MVT::i8)
1111 return false;
1112
1113 // See whether we can avoid an AND in the first operand by converting
1114 // ROSBG to RISBG.
1115 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1116 Opcode = SystemZ::RISBG;
1117 // Prefer RISBGN if available, since it does not clobber CC.
1118 if (Subtarget->hasMiscellaneousExtensions())
1119 Opcode = SystemZ::RISBGN;
1120 }
1121
1122 SDValue Ops[5] = {
1123 convertTo(DL, MVT::i64, Op0),
1124 convertTo(DL, MVT::i64, RxSBG[I].Input),
1125 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1126 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1127 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1128 };
1129 SDValue New = convertTo(
1130 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1131 ReplaceNode(N, New.getNode());
1132 return true;
1133}
1134
1135void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1136 SDValue Op0, uint64_t UpperVal,
1137 uint64_t LowerVal) {
1138 EVT VT = Node->getValueType(0);
1139 SDLoc DL(Node);
1140 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1141 if (Op0.getNode())
1142 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1143
1144 {
1145 // When we haven't passed in Op0, Upper will be a constant. In order to
1146 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1147 // SelectCode first and end up with an opaque machine node. This means that
1148 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1149 // SelectCode.
1150 //
1151 // Note that in the case where Op0 is passed in we could just call
1152 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1153 // the handle at all, but it's fine to do it here.
1154 //
1155 // TODO: This is a pretty hacky way to do this. Can we do something that
1156 // doesn't require a two paragraph explanation?
1157 HandleSDNode Handle(Upper);
1158 SelectCode(Upper.getNode());
1159 Upper = Handle.getValue();
1160 }
1161
1162 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1163 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1164
1165 ReplaceNode(Node, Or.getNode());
1166
1167 SelectCode(Or.getNode());
1168}
1169
1170void SystemZDAGToDAGISel::loadVectorConstant(
1171 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1175 "Bad opcode!");
1176 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1177 EVT VT = Node->getValueType(0);
1178 SDLoc DL(Node);
1180 for (unsigned OpVal : VCI.OpVals)
1181 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1182 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1183
1184 if (VCI.VecVT == VT.getSimpleVT())
1185 ReplaceNode(Node, Op.getNode());
1186 else if (VT.getSizeInBits() == 128) {
1187 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1188 ReplaceNode(Node, BitCast.getNode());
1189 SelectCode(BitCast.getNode());
1190 } else { // float or double
1191 unsigned SubRegIdx =
1192 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1193 ReplaceNode(
1194 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1195 }
1196 SelectCode(Op.getNode());
1197}
1198
1199SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) {
1200 SDNode *ResNode;
1201 assert (VT.getSizeInBits() == 128);
1202
1203 SDValue CP = CurDAG->getTargetConstantPool(
1204 ConstantInt::get(Type::getInt128Ty(*CurDAG->getContext()), Val),
1205 TLI->getPointerTy(CurDAG->getDataLayout()));
1206
1207 EVT PtrVT = CP.getValueType();
1208 SDValue Ops[] = {
1209 SDValue(CurDAG->getMachineNode(SystemZ::LARL, DL, PtrVT, CP), 0),
1210 CurDAG->getTargetConstant(0, DL, PtrVT),
1211 CurDAG->getRegister(0, PtrVT),
1212 CurDAG->getEntryNode()
1213 };
1214 ResNode = CurDAG->getMachineNode(SystemZ::VL, DL, VT, MVT::Other, Ops);
1215
1216 // Annotate ResNode with memory operand information so that MachineInstr
1217 // queries work properly. This e.g. gives the register allocation the
1218 // required information for rematerialization.
1219 MachineFunction& MF = CurDAG->getMachineFunction();
1223
1224 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
1225 return ResNode;
1226}
1227
1228bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1229 SDValue ElemV = N->getOperand(2);
1230 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1231 if (!ElemN)
1232 return false;
1233
1234 unsigned Elem = ElemN->getZExtValue();
1235 EVT VT = N->getValueType(0);
1236 if (Elem >= VT.getVectorNumElements())
1237 return false;
1238
1239 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1240 if (!Load || !Load->hasNUsesOfValue(1, 0))
1241 return false;
1242 if (Load->getMemoryVT().getSizeInBits() !=
1243 Load->getValueType(0).getSizeInBits())
1244 return false;
1245
1246 SDValue Base, Disp, Index;
1247 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1248 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1249 return false;
1250
1251 SDLoc DL(Load);
1252 SDValue Ops[] = {
1253 N->getOperand(0), Base, Disp, Index,
1254 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1255 };
1256 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1257 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1258 ReplaceNode(N, Res);
1259 return true;
1260}
1261
1262bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1263 SDValue Value = Store->getValue();
1264 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1265 return false;
1266 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1267 return false;
1268
1269 SDValue ElemV = Value.getOperand(1);
1270 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1271 if (!ElemN)
1272 return false;
1273
1274 SDValue Vec = Value.getOperand(0);
1275 EVT VT = Vec.getValueType();
1276 unsigned Elem = ElemN->getZExtValue();
1277 if (Elem >= VT.getVectorNumElements())
1278 return false;
1279
1280 SDValue Base, Disp, Index;
1281 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1282 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1283 return false;
1284
1285 SDLoc DL(Store);
1286 SDValue Ops[] = {
1287 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1288 Store->getChain()
1289 };
1290 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1291 return true;
1292}
1293
1294// Check whether or not the chain ending in StoreNode is suitable for doing
1295// the {load; op; store} to modify transformation.
1297 SDValue StoredVal, SelectionDAG *CurDAG,
1298 LoadSDNode *&LoadNode,
1299 SDValue &InputChain) {
1300 // Is the stored value result 0 of the operation?
1301 if (StoredVal.getResNo() != 0)
1302 return false;
1303
1304 // Are there other uses of the loaded value than the operation?
1305 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1306 return false;
1307
1308 // Is the store non-extending and non-indexed?
1309 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1310 return false;
1311
1312 SDValue Load = StoredVal->getOperand(0);
1313 // Is the stored value a non-extending and non-indexed load?
1314 if (!ISD::isNormalLoad(Load.getNode()))
1315 return false;
1316
1317 // Return LoadNode by reference.
1318 LoadNode = cast<LoadSDNode>(Load);
1319
1320 // Is store the only read of the loaded value?
1321 if (!Load.hasOneUse())
1322 return false;
1323
1324 // Is the address of the store the same as the load?
1325 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1326 LoadNode->getOffset() != StoreNode->getOffset())
1327 return false;
1328
1329 // Check if the chain is produced by the load or is a TokenFactor with
1330 // the load output chain as an operand. Return InputChain by reference.
1331 SDValue Chain = StoreNode->getChain();
1332
1333 bool ChainCheck = false;
1334 if (Chain == Load.getValue(1)) {
1335 ChainCheck = true;
1336 InputChain = LoadNode->getChain();
1337 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1338 SmallVector<SDValue, 4> ChainOps;
1339 SmallVector<const SDNode *, 4> LoopWorklist;
1341 const unsigned int Max = 1024;
1342 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1343 SDValue Op = Chain.getOperand(i);
1344 if (Op == Load.getValue(1)) {
1345 ChainCheck = true;
1346 // Drop Load, but keep its chain. No cycle check necessary.
1347 ChainOps.push_back(Load.getOperand(0));
1348 continue;
1349 }
1350 LoopWorklist.push_back(Op.getNode());
1351 ChainOps.push_back(Op);
1352 }
1353
1354 if (ChainCheck) {
1355 // Add the other operand of StoredVal to worklist.
1356 for (SDValue Op : StoredVal->ops())
1357 if (Op.getNode() != LoadNode)
1358 LoopWorklist.push_back(Op.getNode());
1359
1360 // Check if Load is reachable from any of the nodes in the worklist.
1361 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1362 true))
1363 return false;
1364
1365 // Make a new TokenFactor with all the other input chains except
1366 // for the load.
1367 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1368 MVT::Other, ChainOps);
1369 }
1370 }
1371 if (!ChainCheck)
1372 return false;
1373
1374 return true;
1375}
1376
1377// Change a chain of {load; op; store} of the same value into a simple op
1378// through memory of that value, if the uses of the modified value and its
1379// address are suitable.
1380//
1381// The tablegen pattern memory operand pattern is currently not able to match
1382// the case where the CC on the original operation are used.
1383//
1384// See the equivalent routine in X86ISelDAGToDAG for further comments.
1385bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1386 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1387 SDValue StoredVal = StoreNode->getOperand(1);
1388 unsigned Opc = StoredVal->getOpcode();
1389 SDLoc DL(StoreNode);
1390
1391 // Before we try to select anything, make sure this is memory operand size
1392 // and opcode we can handle. Note that this must match the code below that
1393 // actually lowers the opcodes.
1394 EVT MemVT = StoreNode->getMemoryVT();
1395 unsigned NewOpc = 0;
1396 bool NegateOperand = false;
1397 switch (Opc) {
1398 default:
1399 return false;
1400 case SystemZISD::SSUBO:
1401 NegateOperand = true;
1402 [[fallthrough]];
1403 case SystemZISD::SADDO:
1404 if (MemVT == MVT::i32)
1405 NewOpc = SystemZ::ASI;
1406 else if (MemVT == MVT::i64)
1407 NewOpc = SystemZ::AGSI;
1408 else
1409 return false;
1410 break;
1411 case SystemZISD::USUBO:
1412 NegateOperand = true;
1413 [[fallthrough]];
1414 case SystemZISD::UADDO:
1415 if (MemVT == MVT::i32)
1416 NewOpc = SystemZ::ALSI;
1417 else if (MemVT == MVT::i64)
1418 NewOpc = SystemZ::ALGSI;
1419 else
1420 return false;
1421 break;
1422 }
1423
1424 LoadSDNode *LoadNode = nullptr;
1425 SDValue InputChain;
1426 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1427 InputChain))
1428 return false;
1429
1430 SDValue Operand = StoredVal.getOperand(1);
1431 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1432 if (!OperandC)
1433 return false;
1434 auto OperandV = OperandC->getAPIntValue();
1435 if (NegateOperand)
1436 OperandV = -OperandV;
1437 if (OperandV.getSignificantBits() > 8)
1438 return false;
1439 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1440
1441 SDValue Base, Disp;
1442 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1443 return false;
1444
1445 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1447 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1448 CurDAG->setNodeMemRefs(
1449 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1450
1451 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1452 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1453 CurDAG->RemoveDeadNode(Node);
1454 return true;
1455}
1456
1457bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1458 LoadSDNode *Load) const {
1459 // Check that the two memory operands have the same size.
1460 if (Load->getMemoryVT() != Store->getMemoryVT())
1461 return false;
1462
1463 // Volatility stops an access from being decomposed.
1464 if (Load->isVolatile() || Store->isVolatile())
1465 return false;
1466
1467 // There's no chance of overlap if the load is invariant.
1468 if (Load->isInvariant() && Load->isDereferenceable())
1469 return true;
1470
1471 // Otherwise we need to check whether there's an alias.
1472 const Value *V1 = Load->getMemOperand()->getValue();
1473 const Value *V2 = Store->getMemOperand()->getValue();
1474 if (!V1 || !V2)
1475 return false;
1476
1477 // Reject equality.
1478 uint64_t Size = Load->getMemoryVT().getStoreSize();
1479 int64_t End1 = Load->getSrcValueOffset() + Size;
1480 int64_t End2 = Store->getSrcValueOffset() + Size;
1481 if (V1 == V2 && End1 == End2)
1482 return false;
1483
1484 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1485 MemoryLocation(V2, End2, Store->getAAInfo()));
1486}
1487
1488bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1489 auto *Store = cast<StoreSDNode>(N);
1490 auto *Load = cast<LoadSDNode>(Store->getValue());
1491
1492 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1493 // instructions.
1494 uint64_t Size = Load->getMemoryVT().getStoreSize();
1495 if (Size > 1 && Size <= 8) {
1496 // Prefer LHRL, LRL and LGRL.
1497 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1498 return false;
1499 // Prefer STHRL, STRL and STGRL.
1500 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1501 return false;
1502 }
1503
1504 return canUseBlockOperation(Store, Load);
1505}
1506
1507bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1508 unsigned I) const {
1509 auto *StoreA = cast<StoreSDNode>(N);
1510 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1511 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1512 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1513 canUseBlockOperation(StoreA, LoadB);
1514}
1515
1516bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1517
1518 auto *MemAccess = cast<MemSDNode>(N);
1519 auto *LdSt = dyn_cast<LSBaseSDNode>(MemAccess);
1520 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1521 SDValue BasePtr = MemAccess->getBasePtr();
1522 MachineMemOperand *MMO = MemAccess->getMemOperand();
1523 assert(MMO && "Expected a memory operand.");
1524
1525 // The memory access must have a proper alignment and no index register.
1526 // Only load and store nodes have the offset operand (atomic loads do not).
1527 if (MemAccess->getAlign().value() < StoreSize ||
1528 (LdSt && !LdSt->getOffset().isUndef()))
1529 return false;
1530
1531 // The MMO must not have an unaligned offset.
1532 if (MMO->getOffset() % StoreSize != 0)
1533 return false;
1534
1535 // An access to GOT or the Constant Pool is aligned.
1536 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1537 if ((PSV->isGOT() || PSV->isConstantPool()))
1538 return true;
1539
1540 // Check the alignment of a Global Address.
1541 if (BasePtr.getNumOperands())
1542 if (GlobalAddressSDNode *GA =
1543 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1544 // The immediate offset must be aligned.
1545 if (GA->getOffset() % StoreSize != 0)
1546 return false;
1547
1548 // The alignment of the symbol itself must be at least the store size.
1549 const GlobalValue *GV = GA->getGlobal();
1550 const DataLayout &DL = GV->getParent()->getDataLayout();
1551 if (GV->getPointerAlignment(DL).value() < StoreSize)
1552 return false;
1553 }
1554
1555 return true;
1556}
1557
1558ISD::LoadExtType SystemZDAGToDAGISel::getLoadExtType(SDNode *N) const {
1559 ISD::LoadExtType ETy;
1560 if (auto *L = dyn_cast<LoadSDNode>(N))
1561 ETy = L->getExtensionType();
1562 else if (auto *AL = dyn_cast<AtomicSDNode>(N))
1563 ETy = AL->getExtensionType();
1564 else
1565 llvm_unreachable("Unkown load node type.");
1566 return ETy;
1567}
1568
1569void SystemZDAGToDAGISel::Select(SDNode *Node) {
1570 // If we have a custom node, we already have selected!
1571 if (Node->isMachineOpcode()) {
1572 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1573 Node->setNodeId(-1);
1574 return;
1575 }
1576
1577 unsigned Opcode = Node->getOpcode();
1578 switch (Opcode) {
1579 case ISD::OR:
1580 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1581 if (tryRxSBG(Node, SystemZ::ROSBG))
1582 return;
1583 goto or_xor;
1584
1585 case ISD::XOR:
1586 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1587 if (tryRxSBG(Node, SystemZ::RXSBG))
1588 return;
1589 // Fall through.
1590 or_xor:
1591 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1592 // split the operation into two. If both operands here happen to be
1593 // constant, leave this to common code to optimize.
1594 if (Node->getValueType(0) == MVT::i64 &&
1595 Node->getOperand(0).getOpcode() != ISD::Constant)
1596 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1597 uint64_t Val = Op1->getZExtValue();
1598 // Don't split the operation if we can match one of the combined
1599 // logical operations provided by miscellaneous-extensions-3.
1600 if (Subtarget->hasMiscellaneousExtensions3()) {
1601 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1602 // Check whether this expression matches NAND/NOR/NXOR.
1603 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1604 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1605 ChildOpcode == ISD::XOR)
1606 break;
1607 // Check whether this expression matches OR-with-complement
1608 // (or matches an alternate pattern for NXOR).
1609 if (ChildOpcode == ISD::XOR) {
1610 auto Op0 = Node->getOperand(0);
1611 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1612 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1613 break;
1614 }
1615 }
1616 // Don't split an XOR with -1 as LCGR/AGHI is more compact.
1617 if (Opcode == ISD::XOR && Op1->isAllOnes())
1618 break;
1619 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1620 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1621 Val - uint32_t(Val), uint32_t(Val));
1622 return;
1623 }
1624 }
1625 break;
1626
1627 case ISD::AND:
1628 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1629 if (tryRxSBG(Node, SystemZ::RNSBG))
1630 return;
1631 [[fallthrough]];
1632 case ISD::ROTL:
1633 case ISD::SHL:
1634 case ISD::SRL:
1635 case ISD::ZERO_EXTEND:
1636 if (tryRISBGZero(Node))
1637 return;
1638 break;
1639
1640 case ISD::BSWAP:
1641 if (Node->getValueType(0) == MVT::i128) {
1642 SDLoc DL(Node);
1643 SDValue Src = Node->getOperand(0);
1644 Src = CurDAG->getNode(ISD::BITCAST, DL, MVT::v16i8, Src);
1645
1646 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1647 SDNode *Mask = loadPoolVectorConstant(APInt(128, Bytes), MVT::v16i8, DL);
1648 SDValue Ops[] = { Src, Src, SDValue(Mask, 0) };
1649 SDValue Res = SDValue(CurDAG->getMachineNode(SystemZ::VPERM, DL,
1650 MVT::v16i8, Ops), 0);
1651
1652 Res = CurDAG->getNode(ISD::BITCAST, DL, MVT::i128, Res);
1653 SDNode *ResNode = Res.getNode();
1654 ReplaceNode(Node, ResNode);
1655 SelectCode(Src.getNode());
1656 SelectCode(ResNode);
1657 return;
1658 }
1659 break;
1660
1661 case ISD::Constant:
1662 // If this is a 64-bit constant that is out of the range of LLILF,
1663 // LLIHF and LGFI, split it into two 32-bit pieces.
1664 if (Node->getValueType(0) == MVT::i64) {
1665 uint64_t Val = Node->getAsZExtVal();
1666 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1667 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1668 uint32_t(Val));
1669 return;
1670 }
1671 }
1672 if (Node->getValueType(0) == MVT::i128) {
1673 const APInt &Val = Node->getAsAPIntVal();
1675 if (VCI.isVectorConstantLegal(*Subtarget)) {
1676 loadVectorConstant(VCI, Node);
1677 return;
1678 }
1679 // If we can't materialize the constant we need to use a literal pool.
1680 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128, SDLoc(Node));
1681 ReplaceNode(Node, ResNode);
1682 return;
1683 }
1684 break;
1685
1687 SDValue Op0 = Node->getOperand(0);
1688 SDValue Op1 = Node->getOperand(1);
1689 // Prefer to put any load first, so that it can be matched as a
1690 // conditional load. Likewise for constants in range for LOCHI.
1691 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1692 (Subtarget->hasLoadStoreOnCond2() &&
1693 Node->getValueType(0).isInteger() &&
1694 Node->getValueType(0).getSizeInBits() <= 64 &&
1695 Op1.getOpcode() == ISD::Constant &&
1696 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1697 !(Op0.getOpcode() == ISD::Constant &&
1698 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1699 SDValue CCValid = Node->getOperand(2);
1700 SDValue CCMask = Node->getOperand(3);
1701 uint64_t ConstCCValid = CCValid.getNode()->getAsZExtVal();
1702 uint64_t ConstCCMask = CCMask.getNode()->getAsZExtVal();
1703 // Invert the condition.
1704 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1705 SDLoc(Node), CCMask.getValueType());
1706 SDValue Op4 = Node->getOperand(4);
1707 SDNode *UpdatedNode =
1708 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1709 if (UpdatedNode != Node) {
1710 // In case this node already exists then replace Node with it.
1711 ReplaceNode(Node, UpdatedNode);
1712 Node = UpdatedNode;
1713 }
1714 }
1715 break;
1716 }
1717
1719 EVT VT = Node->getValueType(0);
1720 unsigned ElemBitSize = VT.getScalarSizeInBits();
1721 if (ElemBitSize == 32) {
1722 if (tryGather(Node, SystemZ::VGEF))
1723 return;
1724 } else if (ElemBitSize == 64) {
1725 if (tryGather(Node, SystemZ::VGEG))
1726 return;
1727 }
1728 break;
1729 }
1730
1731 case ISD::BUILD_VECTOR: {
1732 auto *BVN = cast<BuildVectorSDNode>(Node);
1734 if (VCI.isVectorConstantLegal(*Subtarget)) {
1735 loadVectorConstant(VCI, Node);
1736 return;
1737 }
1738 break;
1739 }
1740
1741 case ISD::ConstantFP: {
1742 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1743 if (Imm.isZero() || Imm.isNegZero())
1744 break;
1746 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1747 assert(Success && "Expected legal FP immediate");
1748 loadVectorConstant(VCI, Node);
1749 return;
1750 }
1751
1752 case ISD::STORE: {
1753 if (tryFoldLoadStoreIntoMemOperand(Node))
1754 return;
1755 auto *Store = cast<StoreSDNode>(Node);
1756 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1757 if (ElemBitSize == 32) {
1758 if (tryScatter(Store, SystemZ::VSCEF))
1759 return;
1760 } else if (ElemBitSize == 64) {
1761 if (tryScatter(Store, SystemZ::VSCEG))
1762 return;
1763 }
1764 break;
1765 }
1766
1767 case ISD::ATOMIC_STORE: {
1768 auto *AtomOp = cast<AtomicSDNode>(Node);
1769 // Replace the atomic_store with a regular store and select it. This is
1770 // ok since we know all store instructions <= 8 bytes are atomic, and the
1771 // 16 byte case is already handled during lowering.
1772 StoreSDNode *St = cast<StoreSDNode>(CurDAG->getTruncStore(
1773 AtomOp->getChain(), SDLoc(AtomOp), AtomOp->getVal(),
1774 AtomOp->getBasePtr(), AtomOp->getMemoryVT(), AtomOp->getMemOperand()));
1775 assert(St->getMemOperand()->isAtomic() && "Broken MMO.");
1776 SDNode *Chain = St;
1777 // We have to enforce sequential consistency by performing a
1778 // serialization operation after the store.
1779 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1780 Chain = CurDAG->getMachineNode(SystemZ::Serialize, SDLoc(AtomOp),
1781 MVT::Other, SDValue(Chain, 0));
1782 ReplaceNode(Node, Chain);
1783 SelectCode(St);
1784 return;
1785 }
1786 }
1787
1788 SelectCode(Node);
1789}
1790
1791bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1792 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1793 std::vector<SDValue> &OutOps) {
1794 SystemZAddressingMode::AddrForm Form;
1795 SystemZAddressingMode::DispRange DispRange;
1796 SDValue Base, Disp, Index;
1797
1798 switch(ConstraintID) {
1799 default:
1800 llvm_unreachable("Unexpected asm memory constraint");
1801 case InlineAsm::ConstraintCode::i:
1802 case InlineAsm::ConstraintCode::Q:
1803 case InlineAsm::ConstraintCode::ZQ:
1804 // Accept an address with a short displacement, but no index.
1805 Form = SystemZAddressingMode::FormBD;
1806 DispRange = SystemZAddressingMode::Disp12Only;
1807 break;
1808 case InlineAsm::ConstraintCode::R:
1809 case InlineAsm::ConstraintCode::ZR:
1810 // Accept an address with a short displacement and an index.
1811 Form = SystemZAddressingMode::FormBDXNormal;
1812 DispRange = SystemZAddressingMode::Disp12Only;
1813 break;
1814 case InlineAsm::ConstraintCode::S:
1815 case InlineAsm::ConstraintCode::ZS:
1816 // Accept an address with a long displacement, but no index.
1817 Form = SystemZAddressingMode::FormBD;
1818 DispRange = SystemZAddressingMode::Disp20Only;
1819 break;
1820 case InlineAsm::ConstraintCode::T:
1821 case InlineAsm::ConstraintCode::m:
1822 case InlineAsm::ConstraintCode::o:
1823 case InlineAsm::ConstraintCode::p:
1824 case InlineAsm::ConstraintCode::ZT:
1825 // Accept an address with a long displacement and an index.
1826 // m works the same as T, as this is the most general case.
1827 // We don't really have any special handling of "offsettable"
1828 // memory addresses, so just treat o the same as m.
1829 Form = SystemZAddressingMode::FormBDXNormal;
1830 DispRange = SystemZAddressingMode::Disp20Only;
1831 break;
1832 }
1833
1834 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1835 const TargetRegisterClass *TRC =
1836 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1837 SDLoc DL(Base);
1838 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1839
1840 // Make sure that the base address doesn't go into %r0.
1841 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1842 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1843 Base.getOpcode() != ISD::Register) {
1844 Base =
1845 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1846 DL, Base.getValueType(),
1847 Base, RC), 0);
1848 }
1849
1850 // Make sure that the index register isn't assigned to %r0 either.
1851 if (Index.getOpcode() != ISD::Register) {
1852 Index =
1853 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1854 DL, Index.getValueType(),
1855 Index, RC), 0);
1856 }
1857
1858 OutOps.push_back(Base);
1859 OutOps.push_back(Disp);
1860 OutOps.push_back(Index);
1861 return false;
1862 }
1863
1864 return true;
1865}
1866
1867// IsProfitableToFold - Returns true if is profitable to fold the specific
1868// operand node N of U during instruction selection that starts at Root.
1869bool
1870SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1871 SDNode *Root) const {
1872 // We want to avoid folding a LOAD into an ICMP node if as a result
1873 // we would be forced to spill the condition code into a GPR.
1874 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1875 if (!N.hasOneUse() || !U->hasOneUse())
1876 return false;
1877
1878 // The user of the CC value will usually be a CopyToReg into the
1879 // physical CC register, which in turn is glued and chained to the
1880 // actual instruction that uses the CC value. Bail out if we have
1881 // anything else than that.
1882 SDNode *CCUser = *U->use_begin();
1883 SDNode *CCRegUser = nullptr;
1884 if (CCUser->getOpcode() == ISD::CopyToReg ||
1885 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1886 for (auto *U : CCUser->uses()) {
1887 if (CCRegUser == nullptr)
1888 CCRegUser = U;
1889 else if (CCRegUser != U)
1890 return false;
1891 }
1892 }
1893 if (CCRegUser == nullptr)
1894 return false;
1895
1896 // If the actual instruction is a branch, the only thing that remains to be
1897 // checked is whether the CCUser chain is a predecessor of the load.
1898 if (CCRegUser->isMachineOpcode() &&
1899 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1900 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1901
1902 // Otherwise, the instruction may have multiple operands, and we need to
1903 // verify that none of them are a predecessor of the load. This is exactly
1904 // the same check that would be done by common code if the CC setter were
1905 // glued to the CC user, so simply invoke that check here.
1906 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1907 return false;
1908 }
1909
1910 return true;
1911}
1912
1913namespace {
1914// Represents a sequence for extracting a 0/1 value from an IPM result:
1915// (((X ^ XORValue) + AddValue) >> Bit)
1916struct IPMConversion {
1917 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1918 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1919
1920 int64_t XORValue;
1921 int64_t AddValue;
1922 unsigned Bit;
1923};
1924} // end anonymous namespace
1925
1926// Return a sequence for getting a 1 from an IPM result when CC has a
1927// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1928// The handling of CC values outside CCValid doesn't matter.
1929static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1930 // Deal with cases where the result can be taken directly from a bit
1931 // of the IPM result.
1932 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1933 return IPMConversion(0, 0, SystemZ::IPM_CC);
1934 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1935 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1936
1937 // Deal with cases where we can add a value to force the sign bit
1938 // to contain the right value. Putting the bit in 31 means we can
1939 // use SRL rather than RISBG(L), and also makes it easier to get a
1940 // 0/-1 value, so it has priority over the other tests below.
1941 //
1942 // These sequences rely on the fact that the upper two bits of the
1943 // IPM result are zero.
1944 uint64_t TopBit = uint64_t(1) << 31;
1945 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1946 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1947 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1948 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1949 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1952 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1953 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1954 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1955 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1958 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1959
1960 // Next try inverting the value and testing a bit. 0/1 could be
1961 // handled this way too, but we dealt with that case above.
1962 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1963 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1964
1965 // Handle cases where adding a value forces a non-sign bit to contain
1966 // the right value.
1967 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1968 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1969 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1970 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1971
1972 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1973 // can be done by inverting the low CC bit and applying one of the
1974 // sign-based extractions above.
1975 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1976 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1977 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1978 return IPMConversion(1 << SystemZ::IPM_CC,
1979 TopBit - (3 << SystemZ::IPM_CC), 31);
1980 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1983 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1984 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1987 return IPMConversion(1 << SystemZ::IPM_CC,
1988 TopBit - (1 << SystemZ::IPM_CC), 31);
1989
1990 llvm_unreachable("Unexpected CC combination");
1991}
1992
1993SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1994 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1995 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1996 if (!TrueOp || !FalseOp)
1997 return SDValue();
1998 if (FalseOp->getZExtValue() != 0)
1999 return SDValue();
2000 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
2001 return SDValue();
2002
2003 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
2004 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
2005 if (!CCValidOp || !CCMaskOp)
2006 return SDValue();
2007 int CCValid = CCValidOp->getZExtValue();
2008 int CCMask = CCMaskOp->getZExtValue();
2009
2010 SDLoc DL(Node);
2011 SDValue CCReg = Node->getOperand(4);
2012 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
2013 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
2014
2015 if (IPM.XORValue)
2016 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
2017 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
2018
2019 if (IPM.AddValue)
2020 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
2021 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
2022
2023 EVT VT = Node->getValueType(0);
2024 if (VT == MVT::i32 && IPM.Bit == 31) {
2025 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
2026 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
2027 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
2028 } else {
2029 if (VT != MVT::i32)
2030 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
2031
2032 if (TrueOp->getSExtValue() == 1) {
2033 // The SHR/AND sequence should get optimized to an RISBG.
2034 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
2035 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
2036 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
2037 CurDAG->getConstant(1, DL, VT));
2038 } else {
2039 // Sign-extend from IPM.Bit using a pair of shifts.
2040 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
2041 int SraAmt = VT.getSizeInBits() - 1;
2042 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
2043 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
2044 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
2045 CurDAG->getConstant(SraAmt, DL, MVT::i32));
2046 }
2047 }
2048
2049 return Result;
2050}
2051
2052bool SystemZDAGToDAGISel::shouldSelectForReassoc(SDNode *N) const {
2053 EVT VT = N->getValueType(0);
2054 assert(VT.isFloatingPoint() && "Expected FP SDNode");
2055 return N->getFlags().hasAllowReassociation() &&
2056 N->getFlags().hasNoSignedZeros() && Subtarget->hasVector() &&
2057 (VT != MVT::f32 || Subtarget->hasVectorEnhancements1()) &&
2058 !N->isStrictFPOpcode();
2059}
2060
2061void SystemZDAGToDAGISel::PreprocessISelDAG() {
2062 // If we have conditional immediate loads, we always prefer
2063 // using those over an IPM sequence.
2064 if (Subtarget->hasLoadStoreOnCond2())
2065 return;
2066
2067 bool MadeChange = false;
2068
2069 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
2070 E = CurDAG->allnodes_end();
2071 I != E;) {
2072 SDNode *N = &*I++;
2073 if (N->use_empty())
2074 continue;
2075
2076 SDValue Res;
2077 switch (N->getOpcode()) {
2078 default: break;
2080 Res = expandSelectBoolean(N);
2081 break;
2082 }
2083
2084 if (Res) {
2085 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
2086 LLVM_DEBUG(N->dump(CurDAG));
2087 LLVM_DEBUG(dbgs() << "\nNew: ");
2088 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
2089 LLVM_DEBUG(dbgs() << "\n");
2090
2091 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
2092 MadeChange = true;
2093 }
2094 }
2095
2096 if (MadeChange)
2097 CurDAG->RemoveDeadNodes();
2098}
#define Success
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
const HexagonInstrInfo * TII
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
#define PASS_NAME
#define DEBUG_TYPE
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
Class for arbitrary precision integers.
Definition: APInt.h:76
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1498
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
bool isAtomic() const
Returns true if this operation has an atomic ordering requirement of unordered or higher,...
@ MOLoad
The memory access reads data.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:293
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:535
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
static IntegerType * getInt128Ty(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition: Value.cpp:926
self_iterator getIterator()
Definition: ilist_node.h:109
#define UINT64_MAX
Definition: DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:715
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1248
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:240
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1038
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:784
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:904
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:775
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:706
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:536
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:781
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:799
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:681
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:525
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:787
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:516
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1515
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
const unsigned CCMASK_0
Definition: SystemZ.h:27
const unsigned CCMASK_1
Definition: SystemZ.h:28
const unsigned IPM_CC
Definition: SystemZ.h:112
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:186
const unsigned CCMASK_3
Definition: SystemZ.h:30
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:181
const unsigned CCMASK_2
Definition: SystemZ.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition: DWP.cpp:456
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:159
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)