LLVM  14.0.0git
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the SystemZ target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SystemZTargetMachine.h"
14 #include "SystemZISelLowering.h"
17 #include "llvm/Support/Debug.h"
18 #include "llvm/Support/KnownBits.h"
20 
21 using namespace llvm;
22 
23 #define DEBUG_TYPE "systemz-isel"
24 
25 namespace {
26 // Used to build addressing modes.
27 struct SystemZAddressingMode {
28  // The shape of the address.
29  enum AddrForm {
30  // base+displacement
31  FormBD,
32 
33  // base+displacement+index for load and store operands
34  FormBDXNormal,
35 
36  // base+displacement+index for load address operands
37  FormBDXLA,
38 
39  // base+displacement+index+ADJDYNALLOC
40  FormBDXDynAlloc
41  };
42  AddrForm Form;
43 
44  // The type of displacement. The enum names here correspond directly
45  // to the definitions in SystemZOperand.td. We could split them into
46  // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
47  enum DispRange {
48  Disp12Only,
49  Disp12Pair,
50  Disp20Only,
51  Disp20Only128,
52  Disp20Pair
53  };
54  DispRange DR;
55 
56  // The parts of the address. The address is equivalent to:
57  //
58  // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
59  SDValue Base;
60  int64_t Disp;
61  SDValue Index;
62  bool IncludesDynAlloc;
63 
64  SystemZAddressingMode(AddrForm form, DispRange dr)
65  : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
66 
67  // True if the address can have an index register.
68  bool hasIndexField() { return Form != FormBD; }
69 
70  // True if the address can (and must) include ADJDYNALLOC.
71  bool isDynAlloc() { return Form == FormBDXDynAlloc; }
72 
73  void dump(const llvm::SelectionDAG *DAG) {
74  errs() << "SystemZAddressingMode " << this << '\n';
75 
76  errs() << " Base ";
77  if (Base.getNode())
78  Base.getNode()->dump(DAG);
79  else
80  errs() << "null\n";
81 
82  if (hasIndexField()) {
83  errs() << " Index ";
84  if (Index.getNode())
85  Index.getNode()->dump(DAG);
86  else
87  errs() << "null\n";
88  }
89 
90  errs() << " Disp " << Disp;
91  if (IncludesDynAlloc)
92  errs() << " + ADJDYNALLOC";
93  errs() << '\n';
94  }
95 };
96 
97 // Return a mask with Count low bits set.
98 static uint64_t allOnes(unsigned int Count) {
99  assert(Count <= 64);
100  if (Count > 63)
101  return UINT64_MAX;
102  return (uint64_t(1) << Count) - 1;
103 }
104 
105 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
106 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
107 // Rotate (I5). The combined operand value is effectively:
108 //
109 // (or (rotl Input, Rotate), ~Mask)
110 //
111 // for RNSBG and:
112 //
113 // (and (rotl Input, Rotate), Mask)
114 //
115 // otherwise. The output value has BitSize bits, although Input may be
116 // narrower (in which case the upper bits are don't care), or wider (in which
117 // case the result will be truncated as part of the operation).
118 struct RxSBGOperands {
119  RxSBGOperands(unsigned Op, SDValue N)
120  : Opcode(Op), BitSize(N.getValueSizeInBits()),
121  Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
122  Rotate(0) {}
123 
124  unsigned Opcode;
125  unsigned BitSize;
126  uint64_t Mask;
127  SDValue Input;
128  unsigned Start;
129  unsigned End;
130  unsigned Rotate;
131 };
132 
133 class SystemZDAGToDAGISel : public SelectionDAGISel {
134  const SystemZSubtarget *Subtarget;
135 
136  // Used by SystemZOperands.td to create integer constants.
137  inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
138  return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
139  }
140 
141  const SystemZTargetMachine &getTargetMachine() const {
142  return static_cast<const SystemZTargetMachine &>(TM);
143  }
144 
145  const SystemZInstrInfo *getInstrInfo() const {
146  return Subtarget->getInstrInfo();
147  }
148 
149  // Try to fold more of the base or index of AM into AM, where IsBase
150  // selects between the base and index.
151  bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
152 
153  // Try to describe N in AM, returning true on success.
154  bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
155 
156  // Extract individual target operands from matched address AM.
157  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
158  SDValue &Base, SDValue &Disp) const;
159  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
160  SDValue &Base, SDValue &Disp, SDValue &Index) const;
161 
162  // Try to match Addr as a FormBD address with displacement type DR.
163  // Return true on success, storing the base and displacement in
164  // Base and Disp respectively.
165  bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
166  SDValue &Base, SDValue &Disp) const;
167 
168  // Try to match Addr as a FormBDX address with displacement type DR.
169  // Return true on success and if the result had no index. Store the
170  // base and displacement in Base and Disp respectively.
171  bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
172  SDValue &Base, SDValue &Disp) const;
173 
174  // Try to match Addr as a FormBDX* address of form Form with
175  // displacement type DR. Return true on success, storing the base,
176  // displacement and index in Base, Disp and Index respectively.
177  bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
178  SystemZAddressingMode::DispRange DR, SDValue Addr,
179  SDValue &Base, SDValue &Disp, SDValue &Index) const;
180 
181  // PC-relative address matching routines used by SystemZOperands.td.
182  bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
183  if (SystemZISD::isPCREL(Addr.getOpcode())) {
184  Target = Addr.getOperand(0);
185  return true;
186  }
187  return false;
188  }
189 
190  // BD matching routines used by SystemZOperands.td.
191  bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
192  return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
193  }
194  bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
195  return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
196  }
197  bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
198  return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
199  }
200  bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
201  return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
202  }
203 
204  // MVI matching routines used by SystemZOperands.td.
205  bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
206  return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
207  }
208  bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
209  return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
210  }
211 
212  // BDX matching routines used by SystemZOperands.td.
213  bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
214  SDValue &Index) const {
215  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
216  SystemZAddressingMode::Disp12Only,
217  Addr, Base, Disp, Index);
218  }
219  bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
220  SDValue &Index) const {
221  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
222  SystemZAddressingMode::Disp12Pair,
223  Addr, Base, Disp, Index);
224  }
225  bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
226  SDValue &Index) const {
227  return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
228  SystemZAddressingMode::Disp12Only,
229  Addr, Base, Disp, Index);
230  }
231  bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
232  SDValue &Index) const {
233  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
234  SystemZAddressingMode::Disp20Only,
235  Addr, Base, Disp, Index);
236  }
237  bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
238  SDValue &Index) const {
239  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
240  SystemZAddressingMode::Disp20Only128,
241  Addr, Base, Disp, Index);
242  }
243  bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
244  SDValue &Index) const {
245  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
246  SystemZAddressingMode::Disp20Pair,
247  Addr, Base, Disp, Index);
248  }
249  bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
250  SDValue &Index) const {
251  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
252  SystemZAddressingMode::Disp12Pair,
253  Addr, Base, Disp, Index);
254  }
255  bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
256  SDValue &Index) const {
257  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
258  SystemZAddressingMode::Disp20Pair,
259  Addr, Base, Disp, Index);
260  }
261 
262  // Try to match Addr as an address with a base, 12-bit displacement
263  // and index, where the index is element Elem of a vector.
264  // Return true on success, storing the base, displacement and vector
265  // in Base, Disp and Index respectively.
266  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
267  SDValue &Disp, SDValue &Index) const;
268 
269  // Check whether (or Op (and X InsertMask)) is effectively an insertion
270  // of X into bits InsertMask of some Y != Op. Return true if so and
271  // set Op to that Y.
272  bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
273 
274  // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
275  // Return true on success.
276  bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
277 
278  // Try to fold some of RxSBG.Input into other fields of RxSBG.
279  // Return true on success.
280  bool expandRxSBG(RxSBGOperands &RxSBG) const;
281 
282  // Return an undefined value of type VT.
283  SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
284 
285  // Convert N to VT, if it isn't already.
286  SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
287 
288  // Try to implement AND or shift node N using RISBG with the zero flag set.
289  // Return the selected node on success, otherwise return null.
290  bool tryRISBGZero(SDNode *N);
291 
292  // Try to use RISBG or Opcode to implement OR or XOR node N.
293  // Return the selected node on success, otherwise return null.
294  bool tryRxSBG(SDNode *N, unsigned Opcode);
295 
296  // If Op0 is null, then Node is a constant that can be loaded using:
297  //
298  // (Opcode UpperVal LowerVal)
299  //
300  // If Op0 is nonnull, then Node can be implemented using:
301  //
302  // (Opcode (Opcode Op0 UpperVal) LowerVal)
303  void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
304  uint64_t UpperVal, uint64_t LowerVal);
305 
306  void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
307  SDNode *Node);
308 
309  // Try to use gather instruction Opcode to implement vector insertion N.
310  bool tryGather(SDNode *N, unsigned Opcode);
311 
312  // Try to use scatter instruction Opcode to implement store Store.
313  bool tryScatter(StoreSDNode *Store, unsigned Opcode);
314 
315  // Change a chain of {load; op; store} of the same value into a simple op
316  // through memory of that value, if the uses of the modified value and its
317  // address are suitable.
318  bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
319 
320  // Return true if Load and Store are loads and stores of the same size
321  // and are guaranteed not to overlap. Such operations can be implemented
322  // using block (SS-format) instructions.
323  //
324  // Partial overlap would lead to incorrect code, since the block operations
325  // are logically bytewise, even though they have a fast path for the
326  // non-overlapping case. We also need to avoid full overlap (i.e. two
327  // addresses that might be equal at run time) because although that case
328  // would be handled correctly, it might be implemented by millicode.
329  bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
330 
331  // N is a (store (load Y), X) pattern. Return true if it can use an MVC
332  // from Y to X.
333  bool storeLoadCanUseMVC(SDNode *N) const;
334 
335  // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
336  // if A[1 - I] == X and if N can use a block operation like NC from A[I]
337  // to X.
338  bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
339 
340  // Return true if N (a load or a store) fullfills the alignment
341  // requirements for a PC-relative access.
342  bool storeLoadIsAligned(SDNode *N) const;
343 
344  // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
345  SDValue expandSelectBoolean(SDNode *Node);
346 
347 public:
348  SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
349  : SelectionDAGISel(TM, OptLevel) {}
350 
351  bool runOnMachineFunction(MachineFunction &MF) override {
352  const Function &F = MF.getFunction();
353  if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
354  if (F.hasFnAttribute("mnop-mcount"))
355  report_fatal_error("mnop-mcount only supported with fentry-call");
356  if (F.hasFnAttribute("mrecord-mcount"))
357  report_fatal_error("mrecord-mcount only supported with fentry-call");
358  }
359 
360  Subtarget = &MF.getSubtarget<SystemZSubtarget>();
362  }
363 
364  // Override MachineFunctionPass.
365  StringRef getPassName() const override {
366  return "SystemZ DAG->DAG Pattern Instruction Selection";
367  }
368 
369  // Override SelectionDAGISel.
370  void Select(SDNode *Node) override;
371  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
372  std::vector<SDValue> &OutOps) override;
373  bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
374  void PreprocessISelDAG() override;
375 
376  // Include the pieces autogenerated from the target description.
377  #include "SystemZGenDAGISel.inc"
378 };
379 } // end anonymous namespace
380 
382  CodeGenOpt::Level OptLevel) {
383  return new SystemZDAGToDAGISel(TM, OptLevel);
384 }
385 
386 // Return true if Val should be selected as a displacement for an address
387 // with range DR. Here we're interested in the range of both the instruction
388 // described by DR and of any pairing instruction.
389 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
390  switch (DR) {
391  case SystemZAddressingMode::Disp12Only:
392  return isUInt<12>(Val);
393 
394  case SystemZAddressingMode::Disp12Pair:
395  case SystemZAddressingMode::Disp20Only:
396  case SystemZAddressingMode::Disp20Pair:
397  return isInt<20>(Val);
398 
399  case SystemZAddressingMode::Disp20Only128:
400  return isInt<20>(Val) && isInt<20>(Val + 8);
401  }
402  llvm_unreachable("Unhandled displacement range");
403 }
404 
405 // Change the base or index in AM to Value, where IsBase selects
406 // between the base and index.
407 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
408  SDValue Value) {
409  if (IsBase)
410  AM.Base = Value;
411  else
412  AM.Index = Value;
413 }
414 
415 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
416 // where IsBase selects between the base and index. Try to fold the
417 // ADJDYNALLOC into AM.
418 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
419  SDValue Value) {
420  if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
421  changeComponent(AM, IsBase, Value);
422  AM.IncludesDynAlloc = true;
423  return true;
424  }
425  return false;
426 }
427 
428 // The base of AM is equivalent to Base + Index. Try to use Index as
429 // the index register.
430 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
431  SDValue Index) {
432  if (AM.hasIndexField() && !AM.Index.getNode()) {
433  AM.Base = Base;
434  AM.Index = Index;
435  return true;
436  }
437  return false;
438 }
439 
440 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
441 // between the base and index. Try to fold Op1 into AM's displacement.
442 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
443  SDValue Op0, uint64_t Op1) {
444  // First try adjusting the displacement.
445  int64_t TestDisp = AM.Disp + Op1;
446  if (selectDisp(AM.DR, TestDisp)) {
447  changeComponent(AM, IsBase, Op0);
448  AM.Disp = TestDisp;
449  return true;
450  }
451 
452  // We could consider forcing the displacement into a register and
453  // using it as an index, but it would need to be carefully tuned.
454  return false;
455 }
456 
457 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
458  bool IsBase) const {
459  SDValue N = IsBase ? AM.Base : AM.Index;
460  unsigned Opcode = N.getOpcode();
461  if (Opcode == ISD::TRUNCATE) {
462  N = N.getOperand(0);
463  Opcode = N.getOpcode();
464  }
465  if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
466  SDValue Op0 = N.getOperand(0);
467  SDValue Op1 = N.getOperand(1);
468 
469  unsigned Op0Code = Op0->getOpcode();
470  unsigned Op1Code = Op1->getOpcode();
471 
472  if (Op0Code == SystemZISD::ADJDYNALLOC)
473  return expandAdjDynAlloc(AM, IsBase, Op1);
474  if (Op1Code == SystemZISD::ADJDYNALLOC)
475  return expandAdjDynAlloc(AM, IsBase, Op0);
476 
477  if (Op0Code == ISD::Constant)
478  return expandDisp(AM, IsBase, Op1,
479  cast<ConstantSDNode>(Op0)->getSExtValue());
480  if (Op1Code == ISD::Constant)
481  return expandDisp(AM, IsBase, Op0,
482  cast<ConstantSDNode>(Op1)->getSExtValue());
483 
484  if (IsBase && expandIndex(AM, Op0, Op1))
485  return true;
486  }
487  if (Opcode == SystemZISD::PCREL_OFFSET) {
488  SDValue Full = N.getOperand(0);
489  SDValue Base = N.getOperand(1);
490  SDValue Anchor = Base.getOperand(0);
491  uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
492  cast<GlobalAddressSDNode>(Anchor)->getOffset());
493  return expandDisp(AM, IsBase, Base, Offset);
494  }
495  return false;
496 }
497 
498 // Return true if an instruction with displacement range DR should be
499 // used for displacement value Val. selectDisp(DR, Val) must already hold.
500 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
501  assert(selectDisp(DR, Val) && "Invalid displacement");
502  switch (DR) {
503  case SystemZAddressingMode::Disp12Only:
504  case SystemZAddressingMode::Disp20Only:
505  case SystemZAddressingMode::Disp20Only128:
506  return true;
507 
508  case SystemZAddressingMode::Disp12Pair:
509  // Use the other instruction if the displacement is too large.
510  return isUInt<12>(Val);
511 
512  case SystemZAddressingMode::Disp20Pair:
513  // Use the other instruction if the displacement is small enough.
514  return !isUInt<12>(Val);
515  }
516  llvm_unreachable("Unhandled displacement range");
517 }
518 
519 // Return true if Base + Disp + Index should be performed by LA(Y).
520 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
521  // Don't use LA(Y) for constants.
522  if (!Base)
523  return false;
524 
525  // Always use LA(Y) for frame addresses, since we know that the destination
526  // register is almost always (perhaps always) going to be different from
527  // the frame register.
528  if (Base->getOpcode() == ISD::FrameIndex)
529  return true;
530 
531  if (Disp) {
532  // Always use LA(Y) if there is a base, displacement and index.
533  if (Index)
534  return true;
535 
536  // Always use LA if the displacement is small enough. It should always
537  // be no worse than AGHI (and better if it avoids a move).
538  if (isUInt<12>(Disp))
539  return true;
540 
541  // For similar reasons, always use LAY if the constant is too big for AGHI.
542  // LAY should be no worse than AGFI.
543  if (!isInt<16>(Disp))
544  return true;
545  } else {
546  // Don't use LA for plain registers.
547  if (!Index)
548  return false;
549 
550  // Don't use LA for plain addition if the index operand is only used
551  // once. It should be a natural two-operand addition in that case.
552  if (Index->hasOneUse())
553  return false;
554 
555  // Prefer addition if the second operation is sign-extended, in the
556  // hope of using AGF.
557  unsigned IndexOpcode = Index->getOpcode();
558  if (IndexOpcode == ISD::SIGN_EXTEND ||
559  IndexOpcode == ISD::SIGN_EXTEND_INREG)
560  return false;
561  }
562 
563  // Don't use LA for two-operand addition if either operand is only
564  // used once. The addition instructions are better in that case.
565  if (Base->hasOneUse())
566  return false;
567 
568  return true;
569 }
570 
571 // Return true if Addr is suitable for AM, updating AM if so.
572 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
573  SystemZAddressingMode &AM) const {
574  // Start out assuming that the address will need to be loaded separately,
575  // then try to extend it as much as we can.
576  AM.Base = Addr;
577 
578  // First try treating the address as a constant.
579  if (Addr.getOpcode() == ISD::Constant &&
580  expandDisp(AM, true, SDValue(),
581  cast<ConstantSDNode>(Addr)->getSExtValue()))
582  ;
583  // Also see if it's a bare ADJDYNALLOC.
584  else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
585  expandAdjDynAlloc(AM, true, SDValue()))
586  ;
587  else
588  // Otherwise try expanding each component.
589  while (expandAddress(AM, true) ||
590  (AM.Index.getNode() && expandAddress(AM, false)))
591  continue;
592 
593  // Reject cases where it isn't profitable to use LA(Y).
594  if (AM.Form == SystemZAddressingMode::FormBDXLA &&
595  !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
596  return false;
597 
598  // Reject cases where the other instruction in a pair should be used.
599  if (!isValidDisp(AM.DR, AM.Disp))
600  return false;
601 
602  // Make sure that ADJDYNALLOC is included where necessary.
603  if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
604  return false;
605 
606  LLVM_DEBUG(AM.dump(CurDAG));
607  return true;
608 }
609 
610 // Insert a node into the DAG at least before Pos. This will reposition
611 // the node as needed, and will assign it a node ID that is <= Pos's ID.
612 // Note that this does *not* preserve the uniqueness of node IDs!
613 // The selection DAG must no longer depend on their uniqueness when this
614 // function is used.
615 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
616  if (N->getNodeId() == -1 ||
619  DAG->RepositionNode(Pos->getIterator(), N.getNode());
620  // Mark Node as invalid for pruning as after this it may be a successor to a
621  // selected node but otherwise be in the same position of Pos.
622  // Conservatively mark it with the same -abs(Id) to assure node id
623  // invariant is preserved.
624  N->setNodeId(Pos->getNodeId());
626  }
627 }
628 
629 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
630  EVT VT, SDValue &Base,
631  SDValue &Disp) const {
632  Base = AM.Base;
633  if (!Base.getNode())
634  // Register 0 means "no base". This is mostly useful for shifts.
635  Base = CurDAG->getRegister(0, VT);
636  else if (Base.getOpcode() == ISD::FrameIndex) {
637  // Lower a FrameIndex to a TargetFrameIndex.
638  int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
639  Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
640  } else if (Base.getValueType() != VT) {
641  // Truncate values from i64 to i32, for shifts.
642  assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
643  "Unexpected truncation");
644  SDLoc DL(Base);
645  SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
646  insertDAGNode(CurDAG, Base.getNode(), Trunc);
647  Base = Trunc;
648  }
649 
650  // Lower the displacement to a TargetConstant.
651  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
652 }
653 
654 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
655  EVT VT, SDValue &Base,
656  SDValue &Disp,
657  SDValue &Index) const {
658  getAddressOperands(AM, VT, Base, Disp);
659 
660  Index = AM.Index;
661  if (!Index.getNode())
662  // Register 0 means "no index".
663  Index = CurDAG->getRegister(0, VT);
664 }
665 
666 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
668  SDValue &Disp) const {
669  SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
670  if (!selectAddress(Addr, AM))
671  return false;
672 
673  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
674  return true;
675 }
676 
677 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
679  SDValue &Disp) const {
680  SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
681  if (!selectAddress(Addr, AM) || AM.Index.getNode())
682  return false;
683 
684  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
685  return true;
686 }
687 
688 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
689  SystemZAddressingMode::DispRange DR,
691  SDValue &Disp, SDValue &Index) const {
692  SystemZAddressingMode AM(Form, DR);
693  if (!selectAddress(Addr, AM))
694  return false;
695 
696  getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
697  return true;
698 }
699 
700 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
701  SDValue &Base,
702  SDValue &Disp,
703  SDValue &Index) const {
704  SDValue Regs[2];
705  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
706  Regs[0].getNode() && Regs[1].getNode()) {
707  for (unsigned int I = 0; I < 2; ++I) {
708  Base = Regs[I];
709  Index = Regs[1 - I];
710  // We can't tell here whether the index vector has the right type
711  // for the access; the caller needs to do that instead.
712  if (Index.getOpcode() == ISD::ZERO_EXTEND)
713  Index = Index.getOperand(0);
714  if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
715  Index.getOperand(1) == Elem) {
716  Index = Index.getOperand(0);
717  return true;
718  }
719  }
720  }
721  return false;
722 }
723 
724 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
725  uint64_t InsertMask) const {
726  // We're only interested in cases where the insertion is into some operand
727  // of Op, rather than into Op itself. The only useful case is an AND.
728  if (Op.getOpcode() != ISD::AND)
729  return false;
730 
731  // We need a constant mask.
732  auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
733  if (!MaskNode)
734  return false;
735 
736  // It's not an insertion of Op.getOperand(0) if the two masks overlap.
737  uint64_t AndMask = MaskNode->getZExtValue();
738  if (InsertMask & AndMask)
739  return false;
740 
741  // It's only an insertion if all bits are covered or are known to be zero.
742  // The inner check covers all cases but is more expensive.
743  uint64_t Used = allOnes(Op.getValueSizeInBits());
744  if (Used != (AndMask | InsertMask)) {
745  KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
746  if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
747  return false;
748  }
749 
750  Op = Op.getOperand(0);
751  return true;
752 }
753 
754 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
755  uint64_t Mask) const {
756  const SystemZInstrInfo *TII = getInstrInfo();
757  if (RxSBG.Rotate != 0)
758  Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
759  Mask &= RxSBG.Mask;
760  if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
761  RxSBG.Mask = Mask;
762  return true;
763  }
764  return false;
765 }
766 
767 // Return true if any bits of (RxSBG.Input & Mask) are significant.
768 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
769  // Rotate the mask in the same way as RxSBG.Input is rotated.
770  if (RxSBG.Rotate != 0)
771  Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
772  return (Mask & RxSBG.Mask) != 0;
773 }
774 
775 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
776  SDValue N = RxSBG.Input;
777  unsigned Opcode = N.getOpcode();
778  switch (Opcode) {
779  case ISD::TRUNCATE: {
780  if (RxSBG.Opcode == SystemZ::RNSBG)
781  return false;
782  uint64_t BitSize = N.getValueSizeInBits();
783  uint64_t Mask = allOnes(BitSize);
784  if (!refineRxSBGMask(RxSBG, Mask))
785  return false;
786  RxSBG.Input = N.getOperand(0);
787  return true;
788  }
789  case ISD::AND: {
790  if (RxSBG.Opcode == SystemZ::RNSBG)
791  return false;
792 
793  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
794  if (!MaskNode)
795  return false;
796 
797  SDValue Input = N.getOperand(0);
798  uint64_t Mask = MaskNode->getZExtValue();
799  if (!refineRxSBGMask(RxSBG, Mask)) {
800  // If some bits of Input are already known zeros, those bits will have
801  // been removed from the mask. See if adding them back in makes the
802  // mask suitable.
803  KnownBits Known = CurDAG->computeKnownBits(Input);
804  Mask |= Known.Zero.getZExtValue();
805  if (!refineRxSBGMask(RxSBG, Mask))
806  return false;
807  }
808  RxSBG.Input = Input;
809  return true;
810  }
811 
812  case ISD::OR: {
813  if (RxSBG.Opcode != SystemZ::RNSBG)
814  return false;
815 
816  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
817  if (!MaskNode)
818  return false;
819 
820  SDValue Input = N.getOperand(0);
821  uint64_t Mask = ~MaskNode->getZExtValue();
822  if (!refineRxSBGMask(RxSBG, Mask)) {
823  // If some bits of Input are already known ones, those bits will have
824  // been removed from the mask. See if adding them back in makes the
825  // mask suitable.
826  KnownBits Known = CurDAG->computeKnownBits(Input);
827  Mask &= ~Known.One.getZExtValue();
828  if (!refineRxSBGMask(RxSBG, Mask))
829  return false;
830  }
831  RxSBG.Input = Input;
832  return true;
833  }
834 
835  case ISD::ROTL: {
836  // Any 64-bit rotate left can be merged into the RxSBG.
837  if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
838  return false;
839  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
840  if (!CountNode)
841  return false;
842 
843  RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
844  RxSBG.Input = N.getOperand(0);
845  return true;
846  }
847 
848  case ISD::ANY_EXTEND:
849  // Bits above the extended operand are don't-care.
850  RxSBG.Input = N.getOperand(0);
851  return true;
852 
853  case ISD::ZERO_EXTEND:
854  if (RxSBG.Opcode != SystemZ::RNSBG) {
855  // Restrict the mask to the extended operand.
856  unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
857  if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
858  return false;
859 
860  RxSBG.Input = N.getOperand(0);
861  return true;
862  }
864 
865  case ISD::SIGN_EXTEND: {
866  // Check that the extension bits are don't-care (i.e. are masked out
867  // by the final mask).
868  unsigned BitSize = N.getValueSizeInBits();
869  unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
870  if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
871  // In the case where only the sign bit is active, increase Rotate with
872  // the extension width.
873  if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
874  RxSBG.Rotate += (BitSize - InnerBitSize);
875  else
876  return false;
877  }
878 
879  RxSBG.Input = N.getOperand(0);
880  return true;
881  }
882 
883  case ISD::SHL: {
884  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
885  if (!CountNode)
886  return false;
887 
888  uint64_t Count = CountNode->getZExtValue();
889  unsigned BitSize = N.getValueSizeInBits();
890  if (Count < 1 || Count >= BitSize)
891  return false;
892 
893  if (RxSBG.Opcode == SystemZ::RNSBG) {
894  // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
895  // count bits from RxSBG.Input are ignored.
896  if (maskMatters(RxSBG, allOnes(Count)))
897  return false;
898  } else {
899  // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
900  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
901  return false;
902  }
903 
904  RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
905  RxSBG.Input = N.getOperand(0);
906  return true;
907  }
908 
909  case ISD::SRL:
910  case ISD::SRA: {
911  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
912  if (!CountNode)
913  return false;
914 
915  uint64_t Count = CountNode->getZExtValue();
916  unsigned BitSize = N.getValueSizeInBits();
917  if (Count < 1 || Count >= BitSize)
918  return false;
919 
920  if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
921  // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
922  // count bits from RxSBG.Input are ignored.
923  if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
924  return false;
925  } else {
926  // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
927  // which is similar to SLL above.
928  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
929  return false;
930  }
931 
932  RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
933  RxSBG.Input = N.getOperand(0);
934  return true;
935  }
936  default:
937  return false;
938  }
939 }
940 
941 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
942  SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
943  return SDValue(N, 0);
944 }
945 
946 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
947  SDValue N) const {
948  if (N.getValueType() == MVT::i32 && VT == MVT::i64)
949  return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
950  DL, VT, getUNDEF(DL, MVT::i64), N);
951  if (N.getValueType() == MVT::i64 && VT == MVT::i32)
952  return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
953  assert(N.getValueType() == VT && "Unexpected value types");
954  return N;
955 }
956 
957 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
958  SDLoc DL(N);
959  EVT VT = N->getValueType(0);
960  if (!VT.isInteger() || VT.getSizeInBits() > 64)
961  return false;
962  RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
963  unsigned Count = 0;
964  while (expandRxSBG(RISBG))
965  // The widening or narrowing is expected to be free.
966  // Counting widening or narrowing as a saved operation will result in
967  // preferring an R*SBG over a simple shift/logical instruction.
968  if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
969  RISBG.Input.getOpcode() != ISD::TRUNCATE)
970  Count += 1;
971  if (Count == 0)
972  return false;
973 
974  // Prefer to use normal shift instructions over RISBG, since they can handle
975  // all cases and are sometimes shorter.
976  if (Count == 1 && N->getOpcode() != ISD::AND)
977  return false;
978 
979  // Prefer register extensions like LLC over RISBG. Also prefer to start
980  // out with normal ANDs if one instruction would be enough. We can convert
981  // these ANDs into an RISBG later if a three-address instruction is useful.
982  if (RISBG.Rotate == 0) {
983  bool PreferAnd = false;
984  // Prefer AND for any 32-bit and-immediate operation.
985  if (VT == MVT::i32)
986  PreferAnd = true;
987  // As well as for any 64-bit operation that can be implemented via LLC(R),
988  // LLH(R), LLGT(R), or one of the and-immediate instructions.
989  else if (RISBG.Mask == 0xff ||
990  RISBG.Mask == 0xffff ||
991  RISBG.Mask == 0x7fffffff ||
992  SystemZ::isImmLF(~RISBG.Mask) ||
993  SystemZ::isImmHF(~RISBG.Mask))
994  PreferAnd = true;
995  // And likewise for the LLZRGF instruction, which doesn't have a register
996  // to register version.
997  else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
998  if (Load->getMemoryVT() == MVT::i32 &&
999  (Load->getExtensionType() == ISD::EXTLOAD ||
1000  Load->getExtensionType() == ISD::ZEXTLOAD) &&
1001  RISBG.Mask == 0xffffff00 &&
1002  Subtarget->hasLoadAndZeroRightmostByte())
1003  PreferAnd = true;
1004  }
1005  if (PreferAnd) {
1006  // Replace the current node with an AND. Note that the current node
1007  // might already be that same AND, in which case it is already CSE'd
1008  // with it, and we must not call ReplaceNode.
1009  SDValue In = convertTo(DL, VT, RISBG.Input);
1010  SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1011  SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1012  if (N != New.getNode()) {
1013  insertDAGNode(CurDAG, N, Mask);
1014  insertDAGNode(CurDAG, N, New);
1015  ReplaceNode(N, New.getNode());
1016  N = New.getNode();
1017  }
1018  // Now, select the machine opcode to implement this operation.
1019  if (!N->isMachineOpcode())
1020  SelectCode(N);
1021  return true;
1022  }
1023  }
1024 
1025  unsigned Opcode = SystemZ::RISBG;
1026  // Prefer RISBGN if available, since it does not clobber CC.
1027  if (Subtarget->hasMiscellaneousExtensions())
1028  Opcode = SystemZ::RISBGN;
1029  EVT OpcodeVT = MVT::i64;
1030  if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1031  // We can only use the 32-bit instructions if all source bits are
1032  // in the low 32 bits without wrapping, both after rotation (because
1033  // of the smaller range for Start and End) and before rotation
1034  // (because the input value is truncated).
1035  RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1036  ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1037  ((RISBG.End + RISBG.Rotate) & 63) >=
1038  ((RISBG.Start + RISBG.Rotate) & 63)) {
1039  Opcode = SystemZ::RISBMux;
1040  OpcodeVT = MVT::i32;
1041  RISBG.Start &= 31;
1042  RISBG.End &= 31;
1043  }
1044  SDValue Ops[5] = {
1045  getUNDEF(DL, OpcodeVT),
1046  convertTo(DL, OpcodeVT, RISBG.Input),
1047  CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1048  CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1049  CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1050  };
1051  SDValue New = convertTo(
1052  DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1053  ReplaceNode(N, New.getNode());
1054  return true;
1055 }
1056 
1057 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1058  SDLoc DL(N);
1059  EVT VT = N->getValueType(0);
1060  if (!VT.isInteger() || VT.getSizeInBits() > 64)
1061  return false;
1062  // Try treating each operand of N as the second operand of the RxSBG
1063  // and see which goes deepest.
1064  RxSBGOperands RxSBG[] = {
1065  RxSBGOperands(Opcode, N->getOperand(0)),
1066  RxSBGOperands(Opcode, N->getOperand(1))
1067  };
1068  unsigned Count[] = { 0, 0 };
1069  for (unsigned I = 0; I < 2; ++I)
1070  while (expandRxSBG(RxSBG[I]))
1071  // The widening or narrowing is expected to be free.
1072  // Counting widening or narrowing as a saved operation will result in
1073  // preferring an R*SBG over a simple shift/logical instruction.
1074  if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1075  RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1076  Count[I] += 1;
1077 
1078  // Do nothing if neither operand is suitable.
1079  if (Count[0] == 0 && Count[1] == 0)
1080  return false;
1081 
1082  // Pick the deepest second operand.
1083  unsigned I = Count[0] > Count[1] ? 0 : 1;
1084  SDValue Op0 = N->getOperand(I ^ 1);
1085 
1086  // Prefer IC for character insertions from memory.
1087  if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1088  if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1089  if (Load->getMemoryVT() == MVT::i8)
1090  return false;
1091 
1092  // See whether we can avoid an AND in the first operand by converting
1093  // ROSBG to RISBG.
1094  if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1095  Opcode = SystemZ::RISBG;
1096  // Prefer RISBGN if available, since it does not clobber CC.
1097  if (Subtarget->hasMiscellaneousExtensions())
1098  Opcode = SystemZ::RISBGN;
1099  }
1100 
1101  SDValue Ops[5] = {
1102  convertTo(DL, MVT::i64, Op0),
1103  convertTo(DL, MVT::i64, RxSBG[I].Input),
1104  CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1105  CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1106  CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1107  };
1108  SDValue New = convertTo(
1109  DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1110  ReplaceNode(N, New.getNode());
1111  return true;
1112 }
1113 
1114 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1115  SDValue Op0, uint64_t UpperVal,
1116  uint64_t LowerVal) {
1117  EVT VT = Node->getValueType(0);
1118  SDLoc DL(Node);
1119  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1120  if (Op0.getNode())
1121  Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1122 
1123  {
1124  // When we haven't passed in Op0, Upper will be a constant. In order to
1125  // prevent folding back to the large immediate in `Or = getNode(...)` we run
1126  // SelectCode first and end up with an opaque machine node. This means that
1127  // we need to use a handle to keep track of Upper in case it gets CSE'd by
1128  // SelectCode.
1129  //
1130  // Note that in the case where Op0 is passed in we could just call
1131  // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1132  // the handle at all, but it's fine to do it here.
1133  //
1134  // TODO: This is a pretty hacky way to do this. Can we do something that
1135  // doesn't require a two paragraph explanation?
1136  HandleSDNode Handle(Upper);
1137  SelectCode(Upper.getNode());
1138  Upper = Handle.getValue();
1139  }
1140 
1141  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1142  SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1143 
1144  ReplaceNode(Node, Or.getNode());
1145 
1146  SelectCode(Or.getNode());
1147 }
1148 
1149 void SystemZDAGToDAGISel::loadVectorConstant(
1150  const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1152  VCI.Opcode == SystemZISD::REPLICATE ||
1153  VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1154  "Bad opcode!");
1155  assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1156  EVT VT = Node->getValueType(0);
1157  SDLoc DL(Node);
1159  for (unsigned OpVal : VCI.OpVals)
1160  Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1161  SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1162 
1163  if (VCI.VecVT == VT.getSimpleVT())
1164  ReplaceNode(Node, Op.getNode());
1165  else if (VT.getSizeInBits() == 128) {
1166  SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1167  ReplaceNode(Node, BitCast.getNode());
1168  SelectCode(BitCast.getNode());
1169  } else { // float or double
1170  unsigned SubRegIdx =
1171  (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1172  ReplaceNode(
1173  Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1174  }
1175  SelectCode(Op.getNode());
1176 }
1177 
1178 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1179  SDValue ElemV = N->getOperand(2);
1180  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1181  if (!ElemN)
1182  return false;
1183 
1184  unsigned Elem = ElemN->getZExtValue();
1185  EVT VT = N->getValueType(0);
1186  if (Elem >= VT.getVectorNumElements())
1187  return false;
1188 
1189  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1190  if (!Load || !Load->hasNUsesOfValue(1, 0))
1191  return false;
1192  if (Load->getMemoryVT().getSizeInBits() !=
1193  Load->getValueType(0).getSizeInBits())
1194  return false;
1195 
1196  SDValue Base, Disp, Index;
1197  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1198  Index.getValueType() != VT.changeVectorElementTypeToInteger())
1199  return false;
1200 
1201  SDLoc DL(Load);
1202  SDValue Ops[] = {
1203  N->getOperand(0), Base, Disp, Index,
1204  CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1205  };
1206  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1207  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1208  ReplaceNode(N, Res);
1209  return true;
1210 }
1211 
1212 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1213  SDValue Value = Store->getValue();
1214  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1215  return false;
1216  if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1217  return false;
1218 
1219  SDValue ElemV = Value.getOperand(1);
1220  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1221  if (!ElemN)
1222  return false;
1223 
1224  SDValue Vec = Value.getOperand(0);
1225  EVT VT = Vec.getValueType();
1226  unsigned Elem = ElemN->getZExtValue();
1227  if (Elem >= VT.getVectorNumElements())
1228  return false;
1229 
1230  SDValue Base, Disp, Index;
1231  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1232  Index.getValueType() != VT.changeVectorElementTypeToInteger())
1233  return false;
1234 
1235  SDLoc DL(Store);
1236  SDValue Ops[] = {
1237  Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1238  Store->getChain()
1239  };
1240  ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1241  return true;
1242 }
1243 
1244 // Check whether or not the chain ending in StoreNode is suitable for doing
1245 // the {load; op; store} to modify transformation.
1247  SDValue StoredVal, SelectionDAG *CurDAG,
1248  LoadSDNode *&LoadNode,
1249  SDValue &InputChain) {
1250  // Is the stored value result 0 of the operation?
1251  if (StoredVal.getResNo() != 0)
1252  return false;
1253 
1254  // Are there other uses of the loaded value than the operation?
1255  if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1256  return false;
1257 
1258  // Is the store non-extending and non-indexed?
1259  if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1260  return false;
1261 
1262  SDValue Load = StoredVal->getOperand(0);
1263  // Is the stored value a non-extending and non-indexed load?
1264  if (!ISD::isNormalLoad(Load.getNode()))
1265  return false;
1266 
1267  // Return LoadNode by reference.
1268  LoadNode = cast<LoadSDNode>(Load);
1269 
1270  // Is store the only read of the loaded value?
1271  if (!Load.hasOneUse())
1272  return false;
1273 
1274  // Is the address of the store the same as the load?
1275  if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1276  LoadNode->getOffset() != StoreNode->getOffset())
1277  return false;
1278 
1279  // Check if the chain is produced by the load or is a TokenFactor with
1280  // the load output chain as an operand. Return InputChain by reference.
1281  SDValue Chain = StoreNode->getChain();
1282 
1283  bool ChainCheck = false;
1284  if (Chain == Load.getValue(1)) {
1285  ChainCheck = true;
1286  InputChain = LoadNode->getChain();
1287  } else if (Chain.getOpcode() == ISD::TokenFactor) {
1288  SmallVector<SDValue, 4> ChainOps;
1289  SmallVector<const SDNode *, 4> LoopWorklist;
1291  const unsigned int Max = 1024;
1292  for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1293  SDValue Op = Chain.getOperand(i);
1294  if (Op == Load.getValue(1)) {
1295  ChainCheck = true;
1296  // Drop Load, but keep its chain. No cycle check necessary.
1297  ChainOps.push_back(Load.getOperand(0));
1298  continue;
1299  }
1300  LoopWorklist.push_back(Op.getNode());
1301  ChainOps.push_back(Op);
1302  }
1303 
1304  if (ChainCheck) {
1305  // Add the other operand of StoredVal to worklist.
1306  for (SDValue Op : StoredVal->ops())
1307  if (Op.getNode() != LoadNode)
1308  LoopWorklist.push_back(Op.getNode());
1309 
1310  // Check if Load is reachable from any of the nodes in the worklist.
1311  if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1312  true))
1313  return false;
1314 
1315  // Make a new TokenFactor with all the other input chains except
1316  // for the load.
1317  InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1318  MVT::Other, ChainOps);
1319  }
1320  }
1321  if (!ChainCheck)
1322  return false;
1323 
1324  return true;
1325 }
1326 
1327 // Change a chain of {load; op; store} of the same value into a simple op
1328 // through memory of that value, if the uses of the modified value and its
1329 // address are suitable.
1330 //
1331 // The tablegen pattern memory operand pattern is currently not able to match
1332 // the case where the CC on the original operation are used.
1333 //
1334 // See the equivalent routine in X86ISelDAGToDAG for further comments.
1335 bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1336  StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1337  SDValue StoredVal = StoreNode->getOperand(1);
1338  unsigned Opc = StoredVal->getOpcode();
1339  SDLoc DL(StoreNode);
1340 
1341  // Before we try to select anything, make sure this is memory operand size
1342  // and opcode we can handle. Note that this must match the code below that
1343  // actually lowers the opcodes.
1344  EVT MemVT = StoreNode->getMemoryVT();
1345  unsigned NewOpc = 0;
1346  bool NegateOperand = false;
1347  switch (Opc) {
1348  default:
1349  return false;
1350  case SystemZISD::SSUBO:
1351  NegateOperand = true;
1353  case SystemZISD::SADDO:
1354  if (MemVT == MVT::i32)
1355  NewOpc = SystemZ::ASI;
1356  else if (MemVT == MVT::i64)
1357  NewOpc = SystemZ::AGSI;
1358  else
1359  return false;
1360  break;
1361  case SystemZISD::USUBO:
1362  NegateOperand = true;
1364  case SystemZISD::UADDO:
1365  if (MemVT == MVT::i32)
1366  NewOpc = SystemZ::ALSI;
1367  else if (MemVT == MVT::i64)
1368  NewOpc = SystemZ::ALGSI;
1369  else
1370  return false;
1371  break;
1372  }
1373 
1374  LoadSDNode *LoadNode = nullptr;
1375  SDValue InputChain;
1376  if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1377  InputChain))
1378  return false;
1379 
1380  SDValue Operand = StoredVal.getOperand(1);
1381  auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1382  if (!OperandC)
1383  return false;
1384  auto OperandV = OperandC->getAPIntValue();
1385  if (NegateOperand)
1386  OperandV = -OperandV;
1387  if (OperandV.getMinSignedBits() > 8)
1388  return false;
1389  Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1390 
1391  SDValue Base, Disp;
1392  if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1393  return false;
1394 
1395  SDValue Ops[] = { Base, Disp, Operand, InputChain };
1397  CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1398  CurDAG->setNodeMemRefs(
1399  Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1400 
1401  ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1402  ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1403  CurDAG->RemoveDeadNode(Node);
1404  return true;
1405 }
1406 
1407 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1408  LoadSDNode *Load) const {
1409  // Check that the two memory operands have the same size.
1410  if (Load->getMemoryVT() != Store->getMemoryVT())
1411  return false;
1412 
1413  // Volatility stops an access from being decomposed.
1414  if (Load->isVolatile() || Store->isVolatile())
1415  return false;
1416 
1417  // There's no chance of overlap if the load is invariant.
1418  if (Load->isInvariant() && Load->isDereferenceable())
1419  return true;
1420 
1421  // Otherwise we need to check whether there's an alias.
1422  const Value *V1 = Load->getMemOperand()->getValue();
1423  const Value *V2 = Store->getMemOperand()->getValue();
1424  if (!V1 || !V2)
1425  return false;
1426 
1427  // Reject equality.
1428  uint64_t Size = Load->getMemoryVT().getStoreSize();
1429  int64_t End1 = Load->getSrcValueOffset() + Size;
1430  int64_t End2 = Store->getSrcValueOffset() + Size;
1431  if (V1 == V2 && End1 == End2)
1432  return false;
1433 
1434  return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1435  MemoryLocation(V2, End2, Store->getAAInfo()));
1436 }
1437 
1438 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1439  auto *Store = cast<StoreSDNode>(N);
1440  auto *Load = cast<LoadSDNode>(Store->getValue());
1441 
1442  // Prefer not to use MVC if either address can use ... RELATIVE LONG
1443  // instructions.
1444  uint64_t Size = Load->getMemoryVT().getStoreSize();
1445  if (Size > 1 && Size <= 8) {
1446  // Prefer LHRL, LRL and LGRL.
1447  if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1448  return false;
1449  // Prefer STHRL, STRL and STGRL.
1450  if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1451  return false;
1452  }
1453 
1454  return canUseBlockOperation(Store, Load);
1455 }
1456 
1457 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1458  unsigned I) const {
1459  auto *StoreA = cast<StoreSDNode>(N);
1460  auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1461  auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1462  return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1463  canUseBlockOperation(StoreA, LoadB);
1464 }
1465 
1466 bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1467 
1468  auto *MemAccess = cast<LSBaseSDNode>(N);
1469  TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1470  SDValue BasePtr = MemAccess->getBasePtr();
1471  MachineMemOperand *MMO = MemAccess->getMemOperand();
1472  assert(MMO && "Expected a memory operand.");
1473 
1474  // The memory access must have a proper alignment and no index register.
1475  if (MemAccess->getAlignment() < StoreSize ||
1476  !MemAccess->getOffset().isUndef())
1477  return false;
1478 
1479  // The MMO must not have an unaligned offset.
1480  if (MMO->getOffset() % StoreSize != 0)
1481  return false;
1482 
1483  // An access to GOT or the Constant Pool is aligned.
1484  if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1485  if ((PSV->isGOT() || PSV->isConstantPool()))
1486  return true;
1487 
1488  // Check the alignment of a Global Address.
1489  if (BasePtr.getNumOperands())
1490  if (GlobalAddressSDNode *GA =
1491  dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1492  // The immediate offset must be aligned.
1493  if (GA->getOffset() % StoreSize != 0)
1494  return false;
1495 
1496  // The alignment of the symbol itself must be at least the store size.
1497  const GlobalValue *GV = GA->getGlobal();
1498  const DataLayout &DL = GV->getParent()->getDataLayout();
1499  if (GV->getPointerAlignment(DL).value() < StoreSize)
1500  return false;
1501  }
1502 
1503  return true;
1504 }
1505 
1506 void SystemZDAGToDAGISel::Select(SDNode *Node) {
1507  // If we have a custom node, we already have selected!
1508  if (Node->isMachineOpcode()) {
1509  LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1510  Node->setNodeId(-1);
1511  return;
1512  }
1513 
1514  unsigned Opcode = Node->getOpcode();
1515  switch (Opcode) {
1516  case ISD::OR:
1517  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1518  if (tryRxSBG(Node, SystemZ::ROSBG))
1519  return;
1520  goto or_xor;
1521 
1522  case ISD::XOR:
1523  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1524  if (tryRxSBG(Node, SystemZ::RXSBG))
1525  return;
1526  // Fall through.
1527  or_xor:
1528  // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1529  // split the operation into two. If both operands here happen to be
1530  // constant, leave this to common code to optimize.
1531  if (Node->getValueType(0) == MVT::i64 &&
1532  Node->getOperand(0).getOpcode() != ISD::Constant)
1533  if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1534  uint64_t Val = Op1->getZExtValue();
1535  // Don't split the operation if we can match one of the combined
1536  // logical operations provided by miscellaneous-extensions-3.
1537  if (Subtarget->hasMiscellaneousExtensions3()) {
1538  unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1539  // Check whether this expression matches NAND/NOR/NXOR.
1540  if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1541  if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1542  ChildOpcode == ISD::XOR)
1543  break;
1544  // Check whether this expression matches OR-with-complement
1545  // (or matches an alternate pattern for NXOR).
1546  if (ChildOpcode == ISD::XOR) {
1547  auto Op0 = Node->getOperand(0);
1548  if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1549  if (Op0Op1->getZExtValue() == (uint64_t)-1)
1550  break;
1551  }
1552  }
1553  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1554  splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1555  Val - uint32_t(Val), uint32_t(Val));
1556  return;
1557  }
1558  }
1559  break;
1560 
1561  case ISD::AND:
1562  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1563  if (tryRxSBG(Node, SystemZ::RNSBG))
1564  return;
1566  case ISD::ROTL:
1567  case ISD::SHL:
1568  case ISD::SRL:
1569  case ISD::ZERO_EXTEND:
1570  if (tryRISBGZero(Node))
1571  return;
1572  break;
1573 
1574  case ISD::Constant:
1575  // If this is a 64-bit constant that is out of the range of LLILF,
1576  // LLIHF and LGFI, split it into two 32-bit pieces.
1577  if (Node->getValueType(0) == MVT::i64) {
1578  uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1579  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1580  splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1581  uint32_t(Val));
1582  return;
1583  }
1584  }
1585  break;
1586 
1588  SDValue Op0 = Node->getOperand(0);
1589  SDValue Op1 = Node->getOperand(1);
1590  // Prefer to put any load first, so that it can be matched as a
1591  // conditional load. Likewise for constants in range for LOCHI.
1592  if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1593  (Subtarget->hasLoadStoreOnCond2() &&
1594  Node->getValueType(0).isInteger() &&
1595  Op1.getOpcode() == ISD::Constant &&
1596  isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1597  !(Op0.getOpcode() == ISD::Constant &&
1598  isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1599  SDValue CCValid = Node->getOperand(2);
1600  SDValue CCMask = Node->getOperand(3);
1601  uint64_t ConstCCValid =
1602  cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1603  uint64_t ConstCCMask =
1604  cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1605  // Invert the condition.
1606  CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1607  SDLoc(Node), CCMask.getValueType());
1608  SDValue Op4 = Node->getOperand(4);
1609  SDNode *UpdatedNode =
1610  CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1611  if (UpdatedNode != Node) {
1612  // In case this node already exists then replace Node with it.
1613  ReplaceNode(Node, UpdatedNode);
1614  Node = UpdatedNode;
1615  }
1616  }
1617  break;
1618  }
1619 
1620  case ISD::INSERT_VECTOR_ELT: {
1621  EVT VT = Node->getValueType(0);
1622  unsigned ElemBitSize = VT.getScalarSizeInBits();
1623  if (ElemBitSize == 32) {
1624  if (tryGather(Node, SystemZ::VGEF))
1625  return;
1626  } else if (ElemBitSize == 64) {
1627  if (tryGather(Node, SystemZ::VGEG))
1628  return;
1629  }
1630  break;
1631  }
1632 
1633  case ISD::BUILD_VECTOR: {
1634  auto *BVN = cast<BuildVectorSDNode>(Node);
1635  SystemZVectorConstantInfo VCI(BVN);
1636  if (VCI.isVectorConstantLegal(*Subtarget)) {
1637  loadVectorConstant(VCI, Node);
1638  return;
1639  }
1640  break;
1641  }
1642 
1643  case ISD::ConstantFP: {
1644  APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1645  if (Imm.isZero() || Imm.isNegZero())
1646  break;
1647  SystemZVectorConstantInfo VCI(Imm);
1648  bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1649  assert(Success && "Expected legal FP immediate");
1650  loadVectorConstant(VCI, Node);
1651  return;
1652  }
1653 
1654  case ISD::STORE: {
1655  if (tryFoldLoadStoreIntoMemOperand(Node))
1656  return;
1657  auto *Store = cast<StoreSDNode>(Node);
1658  unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1659  if (ElemBitSize == 32) {
1660  if (tryScatter(Store, SystemZ::VSCEF))
1661  return;
1662  } else if (ElemBitSize == 64) {
1663  if (tryScatter(Store, SystemZ::VSCEG))
1664  return;
1665  }
1666  break;
1667  }
1668  }
1669 
1670  SelectCode(Node);
1671 }
1672 
1673 bool SystemZDAGToDAGISel::
1674 SelectInlineAsmMemoryOperand(const SDValue &Op,
1675  unsigned ConstraintID,
1676  std::vector<SDValue> &OutOps) {
1677  SystemZAddressingMode::AddrForm Form;
1678  SystemZAddressingMode::DispRange DispRange;
1679  SDValue Base, Disp, Index;
1680 
1681  switch(ConstraintID) {
1682  default:
1683  llvm_unreachable("Unexpected asm memory constraint");
1686  // Accept an address with a short displacement, but no index.
1687  Form = SystemZAddressingMode::FormBD;
1688  DispRange = SystemZAddressingMode::Disp12Only;
1689  break;
1691  // Accept an address with a short displacement and an index.
1692  Form = SystemZAddressingMode::FormBDXNormal;
1693  DispRange = SystemZAddressingMode::Disp12Only;
1694  break;
1696  // Accept an address with a long displacement, but no index.
1697  Form = SystemZAddressingMode::FormBD;
1698  DispRange = SystemZAddressingMode::Disp20Only;
1699  break;
1703  // Accept an address with a long displacement and an index.
1704  // m works the same as T, as this is the most general case.
1705  // We don't really have any special handling of "offsettable"
1706  // memory addresses, so just treat o the same as m.
1707  Form = SystemZAddressingMode::FormBDXNormal;
1708  DispRange = SystemZAddressingMode::Disp20Only;
1709  break;
1710  }
1711 
1712  if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1713  const TargetRegisterClass *TRC =
1714  Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1715  SDLoc DL(Base);
1716  SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1717 
1718  // Make sure that the base address doesn't go into %r0.
1719  // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1720  if (Base.getOpcode() != ISD::TargetFrameIndex &&
1721  Base.getOpcode() != ISD::Register) {
1722  Base =
1723  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1724  DL, Base.getValueType(),
1725  Base, RC), 0);
1726  }
1727 
1728  // Make sure that the index register isn't assigned to %r0 either.
1729  if (Index.getOpcode() != ISD::Register) {
1730  Index =
1731  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1732  DL, Index.getValueType(),
1733  Index, RC), 0);
1734  }
1735 
1736  OutOps.push_back(Base);
1737  OutOps.push_back(Disp);
1738  OutOps.push_back(Index);
1739  return false;
1740  }
1741 
1742  return true;
1743 }
1744 
1745 // IsProfitableToFold - Returns true if is profitable to fold the specific
1746 // operand node N of U during instruction selection that starts at Root.
1747 bool
1748 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1749  SDNode *Root) const {
1750  // We want to avoid folding a LOAD into an ICMP node if as a result
1751  // we would be forced to spill the condition code into a GPR.
1752  if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1753  if (!N.hasOneUse() || !U->hasOneUse())
1754  return false;
1755 
1756  // The user of the CC value will usually be a CopyToReg into the
1757  // physical CC register, which in turn is glued and chained to the
1758  // actual instruction that uses the CC value. Bail out if we have
1759  // anything else than that.
1760  SDNode *CCUser = *U->use_begin();
1761  SDNode *CCRegUser = nullptr;
1762  if (CCUser->getOpcode() == ISD::CopyToReg ||
1763  cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1764  for (auto *U : CCUser->uses()) {
1765  if (CCRegUser == nullptr)
1766  CCRegUser = U;
1767  else if (CCRegUser != U)
1768  return false;
1769  }
1770  }
1771  if (CCRegUser == nullptr)
1772  return false;
1773 
1774  // If the actual instruction is a branch, the only thing that remains to be
1775  // checked is whether the CCUser chain is a predecessor of the load.
1776  if (CCRegUser->isMachineOpcode() &&
1777  CCRegUser->getMachineOpcode() == SystemZ::BRC)
1778  return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1779 
1780  // Otherwise, the instruction may have multiple operands, and we need to
1781  // verify that none of them are a predecessor of the load. This is exactly
1782  // the same check that would be done by common code if the CC setter were
1783  // glued to the CC user, so simply invoke that check here.
1784  if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1785  return false;
1786  }
1787 
1788  return true;
1789 }
1790 
1791 namespace {
1792 // Represents a sequence for extracting a 0/1 value from an IPM result:
1793 // (((X ^ XORValue) + AddValue) >> Bit)
1794 struct IPMConversion {
1795  IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1796  : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1797 
1798  int64_t XORValue;
1799  int64_t AddValue;
1800  unsigned Bit;
1801 };
1802 } // end anonymous namespace
1803 
1804 // Return a sequence for getting a 1 from an IPM result when CC has a
1805 // value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1806 // The handling of CC values outside CCValid doesn't matter.
1807 static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1808  // Deal with cases where the result can be taken directly from a bit
1809  // of the IPM result.
1810  if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1811  return IPMConversion(0, 0, SystemZ::IPM_CC);
1812  if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1813  return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1814 
1815  // Deal with cases where we can add a value to force the sign bit
1816  // to contain the right value. Putting the bit in 31 means we can
1817  // use SRL rather than RISBG(L), and also makes it easier to get a
1818  // 0/-1 value, so it has priority over the other tests below.
1819  //
1820  // These sequences rely on the fact that the upper two bits of the
1821  // IPM result are zero.
1822  uint64_t TopBit = uint64_t(1) << 31;
1823  if (CCMask == (CCValid & SystemZ::CCMASK_0))
1824  return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1825  if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1826  return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1827  if (CCMask == (CCValid & (SystemZ::CCMASK_0
1829  | SystemZ::CCMASK_2)))
1830  return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1831  if (CCMask == (CCValid & SystemZ::CCMASK_3))
1832  return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1833  if (CCMask == (CCValid & (SystemZ::CCMASK_1
1835  | SystemZ::CCMASK_3)))
1836  return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1837 
1838  // Next try inverting the value and testing a bit. 0/1 could be
1839  // handled this way too, but we dealt with that case above.
1840  if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1841  return IPMConversion(-1, 0, SystemZ::IPM_CC);
1842 
1843  // Handle cases where adding a value forces a non-sign bit to contain
1844  // the right value.
1845  if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1846  return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1847  if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1848  return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1849 
1850  // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1851  // can be done by inverting the low CC bit and applying one of the
1852  // sign-based extractions above.
1853  if (CCMask == (CCValid & SystemZ::CCMASK_1))
1854  return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1855  if (CCMask == (CCValid & SystemZ::CCMASK_2))
1856  return IPMConversion(1 << SystemZ::IPM_CC,
1857  TopBit - (3 << SystemZ::IPM_CC), 31);
1858  if (CCMask == (CCValid & (SystemZ::CCMASK_0
1860  | SystemZ::CCMASK_3)))
1861  return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1862  if (CCMask == (CCValid & (SystemZ::CCMASK_0
1864  | SystemZ::CCMASK_3)))
1865  return IPMConversion(1 << SystemZ::IPM_CC,
1866  TopBit - (1 << SystemZ::IPM_CC), 31);
1867 
1868  llvm_unreachable("Unexpected CC combination");
1869 }
1870 
1871 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1872  auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1873  auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1874  if (!TrueOp || !FalseOp)
1875  return SDValue();
1876  if (FalseOp->getZExtValue() != 0)
1877  return SDValue();
1878  if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1879  return SDValue();
1880 
1881  auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1882  auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1883  if (!CCValidOp || !CCMaskOp)
1884  return SDValue();
1885  int CCValid = CCValidOp->getZExtValue();
1886  int CCMask = CCMaskOp->getZExtValue();
1887 
1888  SDLoc DL(Node);
1889  SDValue CCReg = Node->getOperand(4);
1890  IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1891  SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1892 
1893  if (IPM.XORValue)
1894  Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1895  CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1896 
1897  if (IPM.AddValue)
1898  Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1899  CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1900 
1901  EVT VT = Node->getValueType(0);
1902  if (VT == MVT::i32 && IPM.Bit == 31) {
1903  unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1904  Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1905  CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1906  } else {
1907  if (VT != MVT::i32)
1908  Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1909 
1910  if (TrueOp->getSExtValue() == 1) {
1911  // The SHR/AND sequence should get optimized to an RISBG.
1912  Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1913  CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1914  Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1915  CurDAG->getConstant(1, DL, VT));
1916  } else {
1917  // Sign-extend from IPM.Bit using a pair of shifts.
1918  int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
1919  int SraAmt = VT.getSizeInBits() - 1;
1920  Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
1921  CurDAG->getConstant(ShlAmt, DL, MVT::i32));
1922  Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
1923  CurDAG->getConstant(SraAmt, DL, MVT::i32));
1924  }
1925  }
1926 
1927  return Result;
1928 }
1929 
1930 void SystemZDAGToDAGISel::PreprocessISelDAG() {
1931  // If we have conditional immediate loads, we always prefer
1932  // using those over an IPM sequence.
1933  if (Subtarget->hasLoadStoreOnCond2())
1934  return;
1935 
1936  bool MadeChange = false;
1937 
1938  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
1939  E = CurDAG->allnodes_end();
1940  I != E;) {
1941  SDNode *N = &*I++;
1942  if (N->use_empty())
1943  continue;
1944 
1945  SDValue Res;
1946  switch (N->getOpcode()) {
1947  default: break;
1949  Res = expandSelectBoolean(N);
1950  break;
1951  }
1952 
1953  if (Res) {
1954  LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
1955  LLVM_DEBUG(N->dump(CurDAG));
1956  LLVM_DEBUG(dbgs() << "\nNew: ");
1957  LLVM_DEBUG(Res.getNode()->dump(CurDAG));
1958  LLVM_DEBUG(dbgs() << "\n");
1959 
1960  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1961  MadeChange = true;
1962  }
1963  }
1964 
1965  if (MadeChange)
1966  CurDAG->RemoveDeadNodes();
1967 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
i
i
Definition: README.txt:29
llvm::SystemZISD::UADDO
@ UADDO
Definition: SystemZISelLowering.h:100
llvm::LoadSDNode::getOffset
const SDValue & getOffset() const
Definition: SelectionDAGNodes.h:2320
llvm::SystemZISD::SELECT_CCMASK
@ SELECT_CCMASK
Definition: SystemZISelLowering.h:77
insertDAGNode
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
Definition: SystemZISelDAGToDAG.cpp:615
llvm::StoreSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2350
llvm::TargetRegisterClass::getID
unsigned getID() const
Return the register class ID number.
Definition: TargetRegisterInfo.h:71
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:23
llvm::SystemZISD::USUBO
@ USUBO
Definition: SystemZISelLowering.h:100
llvm::Value::getPointerAlignment
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition: Value.cpp:918
llvm::RecurKind::Or
@ Or
Bitwise or logical OR of integers.
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1088
llvm::HexPrintStyle::Upper
@ Upper
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:633
shouldUseLA
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
Definition: SystemZISelDAGToDAG.cpp:520
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:852
llvm::SystemZ::CCMASK_0
const unsigned CCMASK_0
Definition: SystemZ.h:27
llvm::SystemZISD::ICMP
@ ICMP
Definition: SystemZISelLowering.h:55
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:151
llvm::Function
Definition: Function.h:62
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::KnownBits::Zero
APInt Zero
Definition: KnownBits.h:24
llvm::Target
Target - Wrapper for Target specific information.
Definition: TargetRegistry.h:137
changeComponent
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
Definition: SystemZISelDAGToDAG.cpp:407
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1175
llvm::SystemZ::CCMASK_1
const unsigned CCMASK_1
Definition: SystemZ.h:28
llvm::MachineMemOperand::getOffset
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
Definition: MachineMemOperand.h:227
llvm::dwarf::Form
Form
Definition: Dwarf.h:131
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2765
llvm::SystemZ::CCMASK_3
const unsigned CCMASK_3
Definition: SystemZ.h:30
llvm::createSystemZISelDag
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
Definition: SystemZISelDAGToDAG.cpp:381
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1214
llvm::MemSDNode::getMemoryVT
EVT getMemoryVT() const
Return the type of the in-memory value.
Definition: SelectionDAGNodes.h:1338
llvm::MemSDNode::getChain
const SDValue & getChain() const
Definition: SelectionDAGNodes.h:1361
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:732
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:454
llvm::SystemZISD::ROTATE_MASK
@ ROTATE_MASK
Definition: SystemZISelLowering.h:168
llvm::APFloat::isZero
bool isZero() const
Definition: APFloat.h:1211
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2300
isValidDisp
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
Definition: SystemZISelDAGToDAG.cpp:500
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:128
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:80
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:893
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::SystemZInstrInfo
Definition: SystemZInstrInfo.h:175
llvm::dump
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Definition: SparseBitVector.h:876
llvm::ISD::EXTLOAD
@ EXTLOAD
Definition: ISDOpcodes.h:1350
llvm::SystemZ::isImmLF
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:181
llvm::JumpTable::Full
@ Full
Definition: TargetOptions.h:50
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
selectDisp
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
Definition: SystemZISelDAGToDAG.cpp:389
F
#define F(x, y, z)
Definition: MD5.cpp:55
KnownBits.h
AliasAnalysis.h
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:660
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:628
llvm::InlineAsm::Constraint_T
@ Constraint_T
Definition: InlineAsm.h:259
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1123
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:216
llvm::SDNode::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this node.
Definition: SelectionDAGNodes.h:702
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:729
expandAdjDynAlloc
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
Definition: SystemZISelDAGToDAG.cpp:418
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:747
llvm::InlineAsm::Constraint_Q
@ Constraint_Q
Definition: InlineAsm.h:256
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:309
llvm::KnownBits::One
APInt One
Definition: KnownBits.h:25
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
expandDisp
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
Definition: SystemZISelDAGToDAG.cpp:442
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:735
llvm::ISD::SRA
@ SRA
Definition: ISDOpcodes.h:658
UINT64_MAX
#define UINT64_MAX
Definition: DataTypes.h:77
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:127
llvm::SDNode::uses
iterator_range< use_iterator > uses()
Definition: SelectionDAGNodes.h:791
llvm::StoreSDNode::getOffset
const SDValue & getOffset() const
Definition: SelectionDAGNodes.h:2351
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:145
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
llvm::APInt::getZExtValue
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1467
llvm::codeview::EncodedFramePtrReg::BasePtr
@ BasePtr
llvm::SystemZVectorConstantInfo::isVectorConstantLegal
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)
Definition: SystemZISelLowering.cpp:717
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:632
llvm::PseudoSourceValue
Special value supplied for machine level alias analysis.
Definition: PseudoSourceValue.h:35
llvm::MemSDNode::isNonTemporal
bool isNonTemporal() const
Definition: SelectionDAGNodes.h:1298
isFusableLoadOpStorePattern
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
Definition: SystemZISelDAGToDAG.cpp:1246
llvm::EVT::changeVectorElementTypeToInteger
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:94
llvm::SystemZISD::SSUBO
@ SSUBO
Definition: SystemZISelLowering.h:100
llvm::SystemZVectorConstantInfo::Opcode
unsigned Opcode
Definition: SystemZISelLowering.h:743
llvm::SelectionDAGISel::InvalidateNodeId
static void InvalidateNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1087
llvm::InlineAsm::Constraint_o
@ Constraint_o
Definition: InlineAsm.h:253
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:641
llvm::tgtok::In
@ In
Definition: TGLexer.h:51
llvm::LegacyLegalizeActions::Lower
@ Lower
The operation itself must be expressed in terms of simpler actions on this target.
Definition: LegacyLegalizerInfo.h:58
llvm::APFloat
Definition: APFloat.h:701
llvm::SDNode::use_begin
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Definition: SelectionDAGNodes.h:785
llvm::ISD::Register
@ Register
Definition: ISDOpcodes.h:74
llvm::SDValue::getNumOperands
unsigned getNumOperands() const
Definition: SelectionDAGNodes.h:1127
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::isInt< 32 >
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:373
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:341
SystemZTargetMachine.h
Index
uint32_t Index
Definition: ELFObjHandler.cpp:83
uint64_t
llvm::SystemZVectorConstantInfo
Definition: SystemZISelLowering.h:734
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1342
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:578
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:925
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
maskMatters
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
Definition: SystemZISelDAGToDAG.cpp:768
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:511
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:906
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8311
llvm::SystemZTargetMachine
Definition: SystemZTargetMachine.h:27
llvm::SDNode::dump
void dump() const
Dump this node, for debugging.
Definition: SelectionDAGDumper.cpp:539
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2328
llvm::ISD::ZEXTLOAD
@ ZEXTLOAD
Definition: ISDOpcodes.h:1350
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:44
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::MVT::getSizeInBits
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Definition: MachineValueType.h:864
llvm::SDNode::getNodeId
int getNodeId() const
Return the unique node id.
Definition: SelectionDAGNodes.h:709
llvm::SDValue::getResNo
unsigned getResNo() const
get the index which selects a specific result in the SDNode
Definition: SelectionDAGNodes.h:148
llvm::InlineAsm::Constraint_i
@ Constraint_i
Definition: InlineAsm.h:251
llvm::SelectionDAGISel::getUninvalidatedNodeId
static int getUninvalidatedNodeId(SDNode *N)
Definition: SelectionDAGISel.cpp:1093
llvm::SystemZSubtarget
Definition: SystemZSubtarget.h:33
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:683
llvm::MachineFunction
Definition: MachineFunction.h:241
SelectionDAGISel.h
getIPMConversion
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
Definition: SystemZISelDAGToDAG.cpp:1807
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:2913
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:47
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::EVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:353
allOnes
static uint64_t allOnes(unsigned int Count)
Definition: SystemZInstrInfo.cpp:52
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
expandIndex
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
Definition: SystemZISelDAGToDAG.cpp:430
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
uint32_t
llvm::SDValue::getOperand
const SDValue & getOperand(unsigned i) const
Definition: SelectionDAGNodes.h:1131
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::SystemZISD::REPLICATE
@ REPLICATE
Definition: SystemZISelLowering.h:171
llvm::SDNode::ops
ArrayRef< SDUse > ops() const
Definition: SelectionDAGNodes.h:915
llvm::CodeGenOpt::Level
Level
Definition: CodeGen.h:52
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:290
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:634
llvm::LoadSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2319
llvm::ISD::FrameIndex
@ FrameIndex
Definition: ISDOpcodes.h:80
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:162
llvm::isInt< 16 >
constexpr bool isInt< 16 >(int64_t x)
Definition: MathExtras.h:370
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::SystemZISD::SADDO
@ SADDO
Definition: SystemZISelLowering.h:100
llvm::GlobalAddressSDNode
Definition: SelectionDAGNodes.h:1720
llvm::KnownBits
Definition: KnownBits.h:23
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:607
bit
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional ldr LCPI1_0 ldr ldr tst movne lsr ldr LCPI1_1 and r0 bx lr it saves an instruction and a register It might be profitable to cse MOVi16 if there are lots of bit immediates with the same bottom half Robert Muth started working on an alternate jump table implementation that does not put the tables in line in the text This is more like the llvm default jump table implementation This might be useful sometime Several revisions of patches are on the mailing beginning while CMP sets them like a subtract Therefore to be able to use CMN for comparisons other than the Z bit
Definition: README.txt:584
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:325
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Success
#define Success
Definition: AArch64Disassembler.cpp:266
llvm::TypeSize
Definition: TypeSize.h:416
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:491
llvm::SystemZ::IPM_CC
const unsigned IPM_CC
Definition: SystemZ.h:112
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:46
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::SystemZSubtarget::getInstrInfo
const SystemZInstrInfo * getInstrInfo() const override
Definition: SystemZSubtarget.h:111
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:137
llvm::SystemZ::CCMASK_2
const unsigned CCMASK_2
Definition: SystemZ.h:29
llvm::InlineAsm::Constraint_S
@ Constraint_S
Definition: InlineAsm.h:258
SystemZISelLowering.h
llvm::SelectionDAG::RepositionNode
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
Definition: SelectionDAG.h:1648
llvm::SystemZISD::PCREL_OFFSET
@ PCREL_OFFSET
Definition: SystemZISelLowering.h:51
llvm::ISD::STORE
@ STORE
Definition: ISDOpcodes.h:926
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::SDNode::getMachineOpcode
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
Definition: SelectionDAGNodes.h:688
llvm::SDNode::hasNUsesOfValue
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Definition: SelectionDAG.cpp:10157
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:657
llvm::SelectionDAGISel
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
Definition: SelectionDAGISel.h:39
llvm::SystemZISD::ADJDYNALLOC
@ ADJDYNALLOC
Definition: SystemZISelLowering.h:81
llvm::SDNode::hasPredecessorHelper
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
Definition: SelectionDAGNodes.h:834
N
#define N
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:659
llvm::tgtok::Bit
@ Bit
Definition: TGLexer.h:50
llvm::SystemZISD::isPCREL
bool isPCREL(unsigned Opcode)
Definition: SystemZISelLowering.h:368
llvm::SDValue::getOpcode
unsigned getOpcode() const
Definition: SelectionDAGNodes.h:1119
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:401
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:2951
llvm::ISD::TargetFrameIndex
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:726
raw_ostream.h
llvm::SystemZ::isImmHF
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:186
llvm::SelectionDAGISel::runOnMachineFunction
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Definition: SelectionDAGISel.cpp:416
llvm::APFloat::isNegZero
bool isNegZero() const
Definition: APFloat.h:1227
llvm::ISD::INSERT_VECTOR_ELT
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:500
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::SystemZVectorConstantInfo::OpVals
SmallVector< unsigned, 2 > OpVals
Definition: SystemZISelLowering.h:744
Debug.h
llvm::InlineAsm::Constraint_R
@ Constraint_R
Definition: InlineAsm.h:257
llvm::SystemZISD::IPM
@ IPM
Definition: SystemZISelLowering.h:146
llvm::SystemZVectorConstantInfo::VecVT
MVT VecVT
Definition: SystemZISelLowering.h:745
llvm::InlineAsm::Constraint_m
@ Constraint_m
Definition: InlineAsm.h:252
llvm::MemoryLocation
Representation for a specific memory location.
Definition: MemoryLocation.h:209
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::SystemZISD::BYTE_MASK
@ BYTE_MASK
Definition: SystemZISelLowering.h:162
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:289
llvm::MachineMemOperand::getPseudoValue
const PseudoSourceValue * getPseudoValue() const
Definition: MachineMemOperand.h:213