LLVM  16.0.0git
BPFISelLowering.cpp
Go to the documentation of this file.
1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that BPF uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "BPFISelLowering.h"
15 #include "BPF.h"
16 #include "BPFSubtarget.h"
17 #include "BPFTargetMachine.h"
25 #include "llvm/IR/DiagnosticInfo.h"
27 #include "llvm/Support/Debug.h"
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "bpf-lower"
33 
34 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
35  cl::Hidden, cl::init(false),
36  cl::desc("Expand memcpy into load/store pairs in order"));
37 
38 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
40  DAG.getContext()->diagnose(
41  DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
42 }
43 
44 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
45  SDValue Val) {
47  std::string Str;
48  raw_string_ostream OS(Str);
49  OS << Msg;
50  Val->print(OS);
51  OS.flush();
52  DAG.getContext()->diagnose(
53  DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc()));
54 }
55 
57  const BPFSubtarget &STI)
58  : TargetLowering(TM) {
59 
60  // Set up the register classes.
61  addRegisterClass(MVT::i64, &BPF::GPRRegClass);
62  if (STI.getHasAlu32())
63  addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
64 
65  // Compute derived properties from the register classes
67 
69 
74 
76 
80 
81  // Set unsupported atomic operations as Custom so
82  // we can emit better error messages than fatal error
83  // from selectiondag.
84  for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
85  if (VT == MVT::i32) {
86  if (STI.getHasAlu32())
87  continue;
88  } else {
90  }
91 
97  }
98 
99  for (auto VT : { MVT::i32, MVT::i64 }) {
100  if (VT == MVT::i32 && !STI.getHasAlu32())
101  continue;
102 
116 
120  }
121 
122  if (STI.getHasAlu32()) {
125  STI.getHasJmp32() ? Custom : Promote);
126  }
127 
132 
137 
138  // Extended load operations for i1 types must be promoted
139  for (MVT VT : MVT::integer_valuetypes()) {
143 
147  }
148 
150 
151  // Function alignments
154 
156  // LLVM generic code will try to expand memcpy into load/store pairs at this
157  // stage which is before quite a few IR optimization passes, therefore the
158  // loads and stores could potentially be moved apart from each other which
159  // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
160  // compilers.
161  //
162  // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
163  // of memcpy to later stage in IR optimization pipeline so those load/store
164  // pairs won't be touched and could be kept in order. Hence, we set
165  // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
166  // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
170  MaxLoadsPerMemcmp = 0;
171  } else {
172  // inline memcpy() for kernel to see explicit copy
173  unsigned CommonMaxStores =
175 
176  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
177  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
179  MaxLoadsPerMemcmp = MaxLoadsPerMemcmpOptSize = CommonMaxStores;
180  }
181 
182  // CPU/Feature control
183  HasAlu32 = STI.getHasAlu32();
184  HasJmp32 = STI.getHasJmp32();
185  HasJmpExt = STI.getHasJmpExt();
186 }
187 
189  return false;
190 }
191 
192 bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
193  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
194  return false;
195  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
196  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
197  return NumBits1 > NumBits2;
198 }
199 
200 bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
201  if (!VT1.isInteger() || !VT2.isInteger())
202  return false;
203  unsigned NumBits1 = VT1.getSizeInBits();
204  unsigned NumBits2 = VT2.getSizeInBits();
205  return NumBits1 > NumBits2;
206 }
207 
208 bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
209  if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
210  return false;
211  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
212  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
213  return NumBits1 == 32 && NumBits2 == 64;
214 }
215 
216 bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
217  if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
218  return false;
219  unsigned NumBits1 = VT1.getSizeInBits();
220  unsigned NumBits2 = VT2.getSizeInBits();
221  return NumBits1 == 32 && NumBits2 == 64;
222 }
223 
226  if (Constraint.size() == 1) {
227  switch (Constraint[0]) {
228  default:
229  break;
230  case 'w':
231  return C_RegisterClass;
232  }
233  }
234 
235  return TargetLowering::getConstraintType(Constraint);
236 }
237 
238 std::pair<unsigned, const TargetRegisterClass *>
240  StringRef Constraint,
241  MVT VT) const {
242  if (Constraint.size() == 1)
243  // GCC Constraint Letters
244  switch (Constraint[0]) {
245  case 'r': // GENERAL_REGS
246  return std::make_pair(0U, &BPF::GPRRegClass);
247  case 'w':
248  if (HasAlu32)
249  return std::make_pair(0U, &BPF::GPR32RegClass);
250  break;
251  default:
252  break;
253  }
254 
255  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
256 }
257 
258 void BPFTargetLowering::ReplaceNodeResults(
260  const char *err_msg;
261  uint32_t Opcode = N->getOpcode();
262  switch (Opcode) {
263  default:
264  report_fatal_error("Unhandled custom legalization");
267  case ISD::ATOMIC_LOAD_OR:
269  case ISD::ATOMIC_SWAP:
271  if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
272  err_msg = "Unsupported atomic operations, please use 32/64 bit version";
273  else
274  err_msg = "Unsupported atomic operations, please use 64 bit version";
275  break;
276  }
277 
278  SDLoc DL(N);
279  fail(DL, DAG, err_msg);
280 }
281 
283  switch (Op.getOpcode()) {
284  case ISD::BR_CC:
285  return LowerBR_CC(Op, DAG);
286  case ISD::GlobalAddress:
287  return LowerGlobalAddress(Op, DAG);
288  case ISD::SELECT_CC:
289  return LowerSELECT_CC(Op, DAG);
291  report_fatal_error("Unsupported dynamic stack allocation");
292  default:
293  llvm_unreachable("unimplemented operand");
294  }
295 }
296 
297 // Calling Convention Implementation
298 #include "BPFGenCallingConv.inc"
299 
300 SDValue BPFTargetLowering::LowerFormalArguments(
301  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
303  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
304  switch (CallConv) {
305  default:
306  report_fatal_error("Unsupported calling convention");
307  case CallingConv::C:
308  case CallingConv::Fast:
309  break;
310  }
311 
313  MachineRegisterInfo &RegInfo = MF.getRegInfo();
314 
315  // Assign locations to all of the incoming arguments.
317  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
318  CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
319 
320  for (auto &VA : ArgLocs) {
321  if (VA.isRegLoc()) {
322  // Arguments passed in registers
323  EVT RegVT = VA.getLocVT();
324  MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
325  switch (SimpleTy) {
326  default: {
327  errs() << "LowerFormalArguments Unhandled argument type: "
328  << RegVT.getEVTString() << '\n';
329  llvm_unreachable(nullptr);
330  }
331  case MVT::i32:
332  case MVT::i64:
333  Register VReg = RegInfo.createVirtualRegister(
334  SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
335  RegInfo.addLiveIn(VA.getLocReg(), VReg);
336  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
337 
338  // If this is an value that has been promoted to wider types, insert an
339  // assert[sz]ext to capture this, then truncate to the right size.
340  if (VA.getLocInfo() == CCValAssign::SExt)
341  ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
342  DAG.getValueType(VA.getValVT()));
343  else if (VA.getLocInfo() == CCValAssign::ZExt)
344  ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
345  DAG.getValueType(VA.getValVT()));
346 
347  if (VA.getLocInfo() != CCValAssign::Full)
348  ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
349 
350  InVals.push_back(ArgValue);
351 
352  break;
353  }
354  } else {
355  fail(DL, DAG, "defined with too many args");
356  InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
357  }
358  }
359 
360  if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
361  fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
362  }
363 
364  return Chain;
365 }
366 
367 const unsigned BPFTargetLowering::MaxArgs = 5;
368 
369 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
370  SmallVectorImpl<SDValue> &InVals) const {
371  SelectionDAG &DAG = CLI.DAG;
372  auto &Outs = CLI.Outs;
373  auto &OutVals = CLI.OutVals;
374  auto &Ins = CLI.Ins;
375  SDValue Chain = CLI.Chain;
376  SDValue Callee = CLI.Callee;
377  bool &IsTailCall = CLI.IsTailCall;
378  CallingConv::ID CallConv = CLI.CallConv;
379  bool IsVarArg = CLI.IsVarArg;
381 
382  // BPF target does not support tail call optimization.
383  IsTailCall = false;
384 
385  switch (CallConv) {
386  default:
387  report_fatal_error("Unsupported calling convention");
388  case CallingConv::Fast:
389  case CallingConv::C:
390  break;
391  }
392 
393  // Analyze operands of the call, assigning locations to each operand.
395  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
396 
397  CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
398 
399  unsigned NumBytes = CCInfo.getNextStackOffset();
400 
401  if (Outs.size() > MaxArgs)
402  fail(CLI.DL, DAG, "too many args to ", Callee);
403 
404  for (auto &Arg : Outs) {
405  ISD::ArgFlagsTy Flags = Arg.Flags;
406  if (!Flags.isByVal())
407  continue;
408 
409  fail(CLI.DL, DAG, "pass by value not supported ", Callee);
410  }
411 
412  auto PtrVT = getPointerTy(MF.getDataLayout());
413  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
414 
415  SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
416 
417  // Walk arg assignments
418  for (unsigned i = 0,
419  e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
420  i != e; ++i) {
421  CCValAssign &VA = ArgLocs[i];
422  SDValue Arg = OutVals[i];
423 
424  // Promote the value if needed.
425  switch (VA.getLocInfo()) {
426  default:
427  llvm_unreachable("Unknown loc info");
428  case CCValAssign::Full:
429  break;
430  case CCValAssign::SExt:
431  Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
432  break;
433  case CCValAssign::ZExt:
434  Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
435  break;
436  case CCValAssign::AExt:
437  Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
438  break;
439  }
440 
441  // Push arguments into RegsToPass vector
442  if (VA.isRegLoc())
443  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
444  else
445  llvm_unreachable("call arg pass bug");
446  }
447 
448  SDValue InFlag;
449 
450  // Build a sequence of copy-to-reg nodes chained together with token chain and
451  // flag operands which copy the outgoing args into registers. The InFlag in
452  // necessary since all emitted instructions must be stuck together.
453  for (auto &Reg : RegsToPass) {
454  Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
455  InFlag = Chain.getValue(1);
456  }
457 
458  // If the callee is a GlobalAddress node (quite common, every direct call is)
459  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
460  // Likewise ExternalSymbol -> TargetExternalSymbol.
461  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
462  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
463  G->getOffset(), 0);
464  } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
465  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
466  fail(CLI.DL, DAG, Twine("A call to built-in function '"
467  + StringRef(E->getSymbol())
468  + "' is not supported."));
469  }
470 
471  // Returns a chain & a flag for retval copy to use.
472  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
474  Ops.push_back(Chain);
475  Ops.push_back(Callee);
476 
477  // Add argument registers to the end of the list so that they are
478  // known live into the call.
479  for (auto &Reg : RegsToPass)
480  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
481 
482  if (InFlag.getNode())
483  Ops.push_back(InFlag);
484 
485  Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
486  InFlag = Chain.getValue(1);
487 
488  // Create the CALLSEQ_END node.
489  Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InFlag, CLI.DL);
490  InFlag = Chain.getValue(1);
491 
492  // Handle result values, copying them out of physregs into vregs that we
493  // return.
494  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
495  InVals);
496 }
497 
498 SDValue
499 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
500  bool IsVarArg,
502  const SmallVectorImpl<SDValue> &OutVals,
503  const SDLoc &DL, SelectionDAG &DAG) const {
504  unsigned Opc = BPFISD::RET_FLAG;
505 
506  // CCValAssign - represent the assignment of the return value to a location
509 
510  // CCState - Info about the registers and stack slot.
511  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
512 
513  if (MF.getFunction().getReturnType()->isAggregateType()) {
514  fail(DL, DAG, "only integer returns supported");
515  return DAG.getNode(Opc, DL, MVT::Other, Chain);
516  }
517 
518  // Analize return values.
519  CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
520 
521  SDValue Flag;
522  SmallVector<SDValue, 4> RetOps(1, Chain);
523 
524  // Copy the result values into the output registers.
525  for (unsigned i = 0; i != RVLocs.size(); ++i) {
526  CCValAssign &VA = RVLocs[i];
527  assert(VA.isRegLoc() && "Can only return in registers!");
528 
529  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
530 
531  // Guarantee that all emitted copies are stuck together,
532  // avoiding something bad.
533  Flag = Chain.getValue(1);
534  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
535  }
536 
537  RetOps[0] = Chain; // Update chain.
538 
539  // Add the flag if we have it.
540  if (Flag.getNode())
541  RetOps.push_back(Flag);
542 
543  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
544 }
545 
546 SDValue BPFTargetLowering::LowerCallResult(
547  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
549  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
550 
552  // Assign locations to each value returned by this call.
554  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
555 
556  if (Ins.size() >= 2) {
557  fail(DL, DAG, "only small returns supported");
558  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
559  InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
560  return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
561  }
562 
563  CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
564 
565  // Copy all of the result registers out of their specified physreg.
566  for (auto &Val : RVLocs) {
567  Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
568  Val.getValVT(), InFlag).getValue(1);
569  InFlag = Chain.getValue(2);
570  InVals.push_back(Chain.getValue(0));
571  }
572 
573  return Chain;
574 }
575 
577  switch (CC) {
578  default:
579  break;
580  case ISD::SETULT:
581  case ISD::SETULE:
582  case ISD::SETLT:
583  case ISD::SETLE:
585  std::swap(LHS, RHS);
586  break;
587  }
588 }
589 
590 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
591  SDValue Chain = Op.getOperand(0);
592  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
593  SDValue LHS = Op.getOperand(2);
594  SDValue RHS = Op.getOperand(3);
595  SDValue Dest = Op.getOperand(4);
596  SDLoc DL(Op);
597 
598  if (!getHasJmpExt())
599  NegateCC(LHS, RHS, CC);
600 
601  return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
602  DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
603 }
604 
605 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
606  SDValue LHS = Op.getOperand(0);
607  SDValue RHS = Op.getOperand(1);
608  SDValue TrueV = Op.getOperand(2);
609  SDValue FalseV = Op.getOperand(3);
610  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
611  SDLoc DL(Op);
612 
613  if (!getHasJmpExt())
614  NegateCC(LHS, RHS, CC);
615 
616  SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
617  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
618  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
619 
620  return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
621 }
622 
623 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
624  switch ((BPFISD::NodeType)Opcode) {
626  break;
627  case BPFISD::RET_FLAG:
628  return "BPFISD::RET_FLAG";
629  case BPFISD::CALL:
630  return "BPFISD::CALL";
631  case BPFISD::SELECT_CC:
632  return "BPFISD::SELECT_CC";
633  case BPFISD::BR_CC:
634  return "BPFISD::BR_CC";
635  case BPFISD::Wrapper:
636  return "BPFISD::Wrapper";
637  case BPFISD::MEMCPY:
638  return "BPFISD::MEMCPY";
639  }
640  return nullptr;
641 }
642 
643 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
644  SelectionDAG &DAG) const {
645  auto N = cast<GlobalAddressSDNode>(Op);
646  assert(N->getOffset() == 0 && "Invalid offset for global address");
647 
648  SDLoc DL(Op);
649  const GlobalValue *GV = N->getGlobal();
650  SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
651 
652  return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
653 }
654 
655 unsigned
656 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
657  unsigned Reg, bool isSigned) const {
658  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
660  int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
661  MachineFunction *F = BB->getParent();
662  DebugLoc DL = MI.getDebugLoc();
663 
664  MachineRegisterInfo &RegInfo = F->getRegInfo();
665 
666  if (!isSigned) {
667  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
668  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
669  return PromotedReg0;
670  }
671  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
672  Register PromotedReg1 = RegInfo.createVirtualRegister(RC);
673  Register PromotedReg2 = RegInfo.createVirtualRegister(RC);
674  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
675  BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
676  .addReg(PromotedReg0).addImm(32);
677  BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
678  .addReg(PromotedReg1).addImm(32);
679 
680  return PromotedReg2;
681 }
682 
684 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
686  const {
687  MachineFunction *MF = MI.getParent()->getParent();
689  MachineInstrBuilder MIB(*MF, MI);
690  unsigned ScratchReg;
691 
692  // This function does custom insertion during lowering BPFISD::MEMCPY which
693  // only has two register operands from memcpy semantics, the copy source
694  // address and the copy destination address.
695  //
696  // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
697  // a third scratch register to serve as the destination register of load and
698  // source register of store.
699  //
700  // The scratch register here is with the Define | Dead | EarlyClobber flags.
701  // The EarlyClobber flag has the semantic property that the operand it is
702  // attached to is clobbered before the rest of the inputs are read. Hence it
703  // must be unique among the operands to the instruction. The Define flag is
704  // needed to coerce the machine verifier that an Undef value isn't a problem
705  // as we anyway is loading memory into it. The Dead flag is needed as the
706  // value in scratch isn't supposed to be used by any other instruction.
707  ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
708  MIB.addReg(ScratchReg,
710 
711  return BB;
712 }
713 
716  MachineBasicBlock *BB) const {
717  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
718  DebugLoc DL = MI.getDebugLoc();
719  unsigned Opc = MI.getOpcode();
720  bool isSelectRROp = (Opc == BPF::Select ||
721  Opc == BPF::Select_64_32 ||
722  Opc == BPF::Select_32 ||
723  Opc == BPF::Select_32_64);
724 
725  bool isMemcpyOp = Opc == BPF::MEMCPY;
726 
727 #ifndef NDEBUG
728  bool isSelectRIOp = (Opc == BPF::Select_Ri ||
729  Opc == BPF::Select_Ri_64_32 ||
730  Opc == BPF::Select_Ri_32 ||
731  Opc == BPF::Select_Ri_32_64);
732 
733 
734  assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
735  "Unexpected instr type to insert");
736 #endif
737 
738  if (isMemcpyOp)
739  return EmitInstrWithCustomInserterMemcpy(MI, BB);
740 
741  bool is32BitCmp = (Opc == BPF::Select_32 ||
742  Opc == BPF::Select_32_64 ||
743  Opc == BPF::Select_Ri_32 ||
744  Opc == BPF::Select_Ri_32_64);
745 
746  // To "insert" a SELECT instruction, we actually have to insert the diamond
747  // control-flow pattern. The incoming instruction knows the destination vreg
748  // to set, the condition code register to branch on, the true/false values to
749  // select between, and a branch opcode to use.
750  const BasicBlock *LLVM_BB = BB->getBasicBlock();
751  MachineFunction::iterator I = ++BB->getIterator();
752 
753  // ThisMBB:
754  // ...
755  // TrueVal = ...
756  // jmp_XX r1, r2 goto Copy1MBB
757  // fallthrough --> Copy0MBB
758  MachineBasicBlock *ThisMBB = BB;
759  MachineFunction *F = BB->getParent();
760  MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
761  MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
762 
763  F->insert(I, Copy0MBB);
764  F->insert(I, Copy1MBB);
765  // Update machine-CFG edges by transferring all successors of the current
766  // block to the new block which will contain the Phi node for the select.
767  Copy1MBB->splice(Copy1MBB->begin(), BB,
768  std::next(MachineBasicBlock::iterator(MI)), BB->end());
770  // Next, add the true and fallthrough blocks as its successors.
771  BB->addSuccessor(Copy0MBB);
772  BB->addSuccessor(Copy1MBB);
773 
774  // Insert Branch if Flag
775  int CC = MI.getOperand(3).getImm();
776  int NewCC;
777  switch (CC) {
778 #define SET_NEWCC(X, Y) \
779  case ISD::X: \
780  if (is32BitCmp && HasJmp32) \
781  NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
782  else \
783  NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
784  break
785  SET_NEWCC(SETGT, JSGT);
786  SET_NEWCC(SETUGT, JUGT);
787  SET_NEWCC(SETGE, JSGE);
788  SET_NEWCC(SETUGE, JUGE);
789  SET_NEWCC(SETEQ, JEQ);
790  SET_NEWCC(SETNE, JNE);
791  SET_NEWCC(SETLT, JSLT);
792  SET_NEWCC(SETULT, JULT);
793  SET_NEWCC(SETLE, JSLE);
794  SET_NEWCC(SETULE, JULE);
795  default:
796  report_fatal_error("unimplemented select CondCode " + Twine(CC));
797  }
798 
799  Register LHS = MI.getOperand(1).getReg();
800  bool isSignedCmp = (CC == ISD::SETGT ||
801  CC == ISD::SETGE ||
802  CC == ISD::SETLT ||
803  CC == ISD::SETLE);
804 
805  // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
806  // to be promoted, however if the 32-bit comparison operands are destination
807  // registers then they are implicitly zero-extended already, there is no
808  // need of explicit zero-extend sequence for them.
809  //
810  // We simply do extension for all situations in this method, but we will
811  // try to remove those unnecessary in BPFMIPeephole pass.
812  if (is32BitCmp && !HasJmp32)
813  LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
814 
815  if (isSelectRROp) {
816  Register RHS = MI.getOperand(2).getReg();
817 
818  if (is32BitCmp && !HasJmp32)
819  RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
820 
821  BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
822  } else {
823  int64_t imm32 = MI.getOperand(2).getImm();
824  // Check before we build J*_ri instruction.
825  assert (isInt<32>(imm32));
826  BuildMI(BB, DL, TII.get(NewCC))
827  .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
828  }
829 
830  // Copy0MBB:
831  // %FalseValue = ...
832  // # fallthrough to Copy1MBB
833  BB = Copy0MBB;
834 
835  // Update machine-CFG edges
836  BB->addSuccessor(Copy1MBB);
837 
838  // Copy1MBB:
839  // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
840  // ...
841  BB = Copy1MBB;
842  BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
843  .addReg(MI.getOperand(5).getReg())
844  .addMBB(Copy0MBB)
845  .addReg(MI.getOperand(4).getReg())
846  .addMBB(ThisMBB);
847 
848  MI.eraseFromParent(); // The pseudo instruction is gone now.
849  return BB;
850 }
851 
853  EVT VT) const {
854  return getHasAlu32() ? MVT::i32 : MVT::i64;
855 }
856 
858  EVT VT) const {
859  return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
860 }
861 
862 bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
863  const AddrMode &AM, Type *Ty,
864  unsigned AS,
865  Instruction *I) const {
866  // No global is ever allowed as a base.
867  if (AM.BaseGV)
868  return false;
869 
870  switch (AM.Scale) {
871  case 0: // "r+i" or just "i", depending on HasBaseReg.
872  break;
873  case 1:
874  if (!AM.HasBaseReg) // allow "r+i".
875  break;
876  return false; // disallow "r+r" or "r+r+i".
877  default:
878  return false;
879  }
880 
881  return true;
882 }
llvm::CCValAssign::getLocVT
MVT getLocVT() const
Definition: CallingConvLower.h:151
llvm::MachineRegisterInfo::addLiveIn
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Definition: MachineRegisterInfo.h:959
i
i
Definition: README.txt:29
llvm::ISD::SETUGE
@ SETUGE
Definition: ISDOpcodes.h:1437
llvm::TargetLoweringBase::MaxStoresPerMemsetOptSize
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3413
llvm::CCValAssign::ZExt
@ ZExt
Definition: CallingConvLower.h:36
ValueTypes.h
llvm::BPFTargetLowering::getTargetNodeName
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Definition: BPFISelLowering.cpp:623
llvm::SelectionDAG::getCALLSEQ_START
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:988
llvm::ISD::SETLE
@ SETLE
Definition: ISDOpcodes.h:1448
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
BPFSubtarget.h
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::CCValAssign::Full
@ Full
Definition: CallingConvLower.h:34
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::SETGT
@ SETGT
Definition: ISDOpcodes.h:1445
llvm::ISD::SETNE
@ SETNE
Definition: ISDOpcodes.h:1449
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:156
llvm::TargetLowering::ConstraintType
ConstraintType
Definition: TargetLowering.h:4554
llvm::ISD::BR_JT
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:991
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
llvm::DiagnosticInfoUnsupported
Diagnostic information for unsupported feature in backend.
Definition: DiagnosticInfo.h:1008
llvm::BPFSubtarget::getHasJmp32
bool getHasJmp32() const
Definition: BPFSubtarget.h:72
llvm::CCState
CCState - This class holds information needed while lowering arguments and return values.
Definition: CallingConvLower.h:189
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:767
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:700
llvm::ISD::DYNAMIC_STACKALLOC
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:976
llvm::BPFISD::MEMCPY
@ MEMCPY
Definition: BPFISelLowering.h:31
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:708
llvm::SelectionDAG::getValueType
SDValue getValueType(EVT)
Definition: SelectionDAG.cpp:1858
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:629
llvm::BPFSubtarget::getRegisterInfo
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:86
llvm::TargetLoweringBase::MaxStoresPerMemset
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
Definition: TargetLowering.h:3411
BPFISelLowering.h
llvm::BPFTargetLowering::getConstraintType
BPFTargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
Definition: BPFISelLowering.cpp:225
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::ISD::SETEQ
@ SETEQ
Definition: ISDOpcodes.h:1444
llvm::ISD::STACKRESTORE
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1057
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9378
llvm::BPFTargetLowering::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFISelLowering.h:62
llvm::BPFTargetLowering::getHasAlu32
bool getHasAlu32() const
Definition: BPFISelLowering.h:60
ErrorHandling.h
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:282
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:237
llvm::ISD::SETULE
@ SETULE
Definition: ISDOpcodes.h:1439
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::ISD::SHL_PARTS
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:749
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
llvm::ARMISD::MEMCPY
@ MEMCPY
Definition: ARMISelLowering.h:310
llvm::TargetLoweringBase::setMinFunctionAlignment
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
Definition: TargetLowering.h:2506
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:772
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:891
llvm::ISD::EXTLOAD
@ EXTLOAD
Definition: ISDOpcodes.h:1404
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
BPF.h
llvm::BPFISD::NodeType
NodeType
Definition: BPFISelLowering.h:24
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:479
llvm::ISD::ATOMIC_LOAD_OR
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1189
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::ISD::BRCOND
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1001
MachineRegisterInfo.h
llvm::BPFTargetLowering::getRegForInlineAsmConstraint
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
Definition: BPFISelLowering.cpp:239
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2121
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::ISD::BRIND
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:987
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:694
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
llvm::MVT::integer_valuetypes
static auto integer_valuetypes()
Definition: MachineValueType.h:1519
llvm::BPFISD::BR_CC
@ BR_CC
Definition: BPFISelLowering.h:29
llvm::BPFISD::FIRST_NUMBER
@ FIRST_NUMBER
Definition: BPFISelLowering.h:25
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1008
llvm::BPFTargetLowering::getSetCCResultType
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
Definition: BPFISelLowering.cpp:852
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::TargetLowering::CallLoweringInfo::IsVarArg
bool IsVarArg
Definition: TargetLowering.h:4143
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:667
llvm::CCValAssign::AExt
@ AExt
Definition: CallingConvLower.h:37
llvm::ISD::GlobalAddress
@ GlobalAddress
Definition: ISDOpcodes.h:78
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
llvm::ExternalSymbolSDNode
Definition: SelectionDAGNodes.h:2237
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:98
llvm::CCValAssign
CCValAssign - Represent assignment of one arg/retval to a location.
Definition: CallingConvLower.h:31
llvm::BPFISD::SELECT_CC
@ SELECT_CC
Definition: BPFISelLowering.h:28
llvm::ISD::CTLZ
@ CTLZ
Definition: ISDOpcodes.h:702
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:713
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:763
llvm::ISD::SETGE
@ SETGE
Definition: ISDOpcodes.h:1446
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::CCValAssign::getLocReg
Register getLocReg() const
Definition: CallingConvLower.h:148
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:781
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1514
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3490
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:769
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:341
llvm::BPFTargetLowering::BPFTargetLowering
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
Definition: BPFISelLowering.cpp:56
llvm::ISD::UDIVREM
@ UDIVREM
Definition: ISDOpcodes.h:256
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::TargetLoweringBase::addRegisterClass
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
Definition: TargetLowering.h:2343
llvm::TargetLoweringBase::MaxLoadsPerMemcmpOptSize
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3449
llvm::ISD::ATOMIC_LOAD_AND
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1187
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::Instruction
Definition: Instruction.h:42
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:147
llvm::BPFTargetLowering::EmitInstrWithCustomInserter
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: BPFISelLowering.cpp:715
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::raw_ostream::flush
void flush()
Definition: raw_ostream.h:186
llvm::TargetLoweringBase::MaxStoresPerMemcpy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
Definition: TargetLowering.h:3426
llvm::CCValAssign::getLocInfo
LocInfo getLocInfo() const
Definition: CallingConvLower.h:153
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::CCValAssign::isRegLoc
bool isRegLoc() const
Definition: CallingConvLower.h:143
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::ISD::SETUGT
@ SETUGT
Definition: ISDOpcodes.h:1436
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:715
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:210
llvm::TargetLowering::CallLoweringInfo::Outs
SmallVector< ISD::OutputArg, 32 > Outs
Definition: TargetLowering.h:4166
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:200
llvm::TargetLoweringBase::setOperationAction
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
Definition: TargetLowering.h:2360
llvm::TargetLoweringBase::MaxStoresPerMemcpyOptSize
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3428
llvm::cl::opt< bool >
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::BPFTargetLowering::LowerOperation
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Definition: BPFISelLowering.cpp:282
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:180
llvm::CCValAssign::SExt
@ SExt
Definition: CallingConvLower.h:35
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1176
llvm::TargetLowering::CallLoweringInfo::Chain
SDValue Chain
Definition: TargetLowering.h:4139
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::TargetLoweringBase::Promote
@ Promote
Definition: TargetLowering.h:197
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:793
llvm::TargetLowering::CallLoweringInfo::CallConv
CallingConv::ID CallConv
Definition: TargetLowering.h:4160
llvm::TargetLoweringBase::setStackPointerRegisterToSaveRestore
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
Definition: TargetLowering.h:2309
fail
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
Definition: BPFISelLowering.cpp:38
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:709
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9072
llvm::TargetLoweringBase::setPrefFunctionAlignment
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
Definition: TargetLowering.h:2512
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::ISD::ATOMIC_LOAD_ADD
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1185
llvm::TargetLowering::CallLoweringInfo::DL
SDLoc DL
Definition: TargetLowering.h:4164
llvm::Function::hasStructRetAttr
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:626
llvm::ISD::ZEXTLOAD
@ ZEXTLOAD
Definition: ISDOpcodes.h:1404
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:46
llvm::TargetLowering::CallLoweringInfo
This structure contains all information that is necessary for lowering calls.
Definition: TargetLowering.h:4138
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::BPFSubtarget::getSelectionDAGInfo
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:83
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
llvm::ISD::MULHS
@ MULHS
Definition: ISDOpcodes.h:638
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::ISD::SETULT
@ SETULT
Definition: ISDOpcodes.h:1438
llvm::SDNode::print
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
Definition: SelectionDAGDumper.cpp:1060
BPFExpandMemcpyInOrder
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1424
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::ISD::SRA_PARTS
@ SRA_PARTS
Definition: ISDOpcodes.h:750
NegateCC
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
Definition: BPFISelLowering.cpp:576
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::SelectionDAG::getCALLSEQ_END
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
Definition: SelectionDAG.h:1000
llvm::TargetLowering::CallLoweringInfo::Ins
SmallVector< ISD::InputArg, 32 > Ins
Definition: TargetLowering.h:4168
llvm::TargetLowering::CallLoweringInfo::DAG
SelectionDAG & DAG
Definition: TargetLowering.h:4163
llvm::BPFSelectionDAGInfo::getCommonMaxStoresPerMemFunc
unsigned getCommonMaxStoresPerMemFunc() const
Definition: BPFSelectionDAGInfo.h:29
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1009
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ISD::SREM
@ SREM
Definition: ISDOpcodes.h:244
llvm::ISD::UMUL_LOHI
@ UMUL_LOHI
Definition: ISDOpcodes.h:251
uint32_t
llvm::ISD::ArgFlagsTy
Definition: TargetCallingConv.h:27
llvm::TargetLoweringBase::MaxStoresPerMemmoveOptSize
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3463
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
AddrMode
AddrMode
Definition: MSP430Disassembler.cpp:142
llvm::BPFISD::CALL
@ CALL
Definition: BPFISelLowering.h:27
CC
auto CC
Definition: RISCVRedundantCopyElimination.cpp:79
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
llvm::TargetLowering::getRegForInlineAsmConstraint
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Definition: TargetLowering.cpp:5254
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1184
llvm::ISD::SEXTLOAD
@ SEXTLOAD
Definition: ISDOpcodes.h:1404
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::TargetLowering::C_RegisterClass
@ C_RegisterClass
Definition: TargetLowering.h:4556
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::AMDGPU::SendMsg::Msg
const CustomOperand< const MCSubtargetInfo & > Msg[]
Definition: AMDGPUAsmUtils.cpp:39
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:187
SET_NEWCC
#define SET_NEWCC(X, Y)
llvm::EVT::getEVTString
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition: ValueTypes.cpp:152
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:164
llvm::StringRef::size
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
llvm::TargetLowering::CallLoweringInfo::IsTailCall
bool IsTailCall
Definition: TargetLowering.h:4154
llvm::ISD::SETLT
@ SETLT
Definition: ISDOpcodes.h:1447
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:248
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::BPFTargetLowering::getScalarShiftAmountTy
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
Definition: BPFISelLowering.cpp:857
llvm::TargetLoweringBase::setLoadExtAction
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
Definition: TargetLowering.h:2377
llvm::GlobalAddressSDNode
Definition: SelectionDAGNodes.h:1759
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:623
CallingConvLower.h
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::ISD::getSetCCSwappedOperands
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
Definition: SelectionDAG.cpp:534
DiagnosticInfo.h
llvm::BPFSubtarget::getHasAlu32
bool getHasAlu32() const
Definition: BPFSubtarget.h:73
llvm::TargetLoweringBase::Custom
@ Custom
Definition: TargetLowering.h:200
llvm::SelectionDAG::getTargetExternalSymbol
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.cpp:1889
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:48
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::TargetLoweringBase::ZeroOrOneBooleanContent
@ ZeroOrOneBooleanContent
Definition: TargetLowering.h:232
llvm::ISD::SRL_PARTS
@ SRL_PARTS
Definition: ISDOpcodes.h:751
llvm::BPFISD::Wrapper
@ Wrapper
Definition: BPFISelLowering.h:30
llvm::TargetLoweringBase::setBooleanContents
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
Definition: TargetLowering.h:2277
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:52
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:357
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:305
MachineInstrBuilder.h
llvm::TargetLoweringBase::Expand
@ Expand
Definition: TargetLowering.h:198
llvm::BPFTargetLowering::isOffsetFoldingLegal
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
Definition: BPFISelLowering.cpp:188
llvm::CallingConv::Fast
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
N
#define N
llvm::TargetLoweringBase::computeRegisterProperties
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
Definition: TargetLoweringBase.cpp:1284
llvm::ISD::CTTZ
@ CTTZ
Definition: ISDOpcodes.h:701
llvm::TargetLoweringBase::getRegClassFor
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
Definition: TargetLowering.h:896
llvm::MachineFunction::getDataLayout
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Definition: MachineFunction.cpp:285
llvm::MipsISD::Ins
@ Ins
Definition: MipsISelLowering.h:160
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:637
llvm::TargetLowering::CallLoweringInfo::OutVals
SmallVector< SDValue, 32 > OutVals
Definition: TargetLowering.h:4167
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:901
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::TargetLowering::CallLoweringInfo::Callee
SDValue Callee
Definition: TargetLowering.h:4161
llvm::BPFSubtarget::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFSubtarget.h:71
llvm::MVT::i16
@ i16
Definition: MachineValueType.h:47
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:466
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::omp::RTLDependInfoFields::Flags
@ Flags
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:276
llvm::BPFISD::RET_FLAG
@ RET_FLAG
Definition: BPFISelLowering.h:26
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
llvm::TargetLoweringBase::MaxLoadsPerMemcmp
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
Definition: TargetLowering.h:3447
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:413
llvm::ISD::ATOMIC_LOAD_XOR
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1190
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:760
raw_ostream.h
MachineFunction.h
DiagnosticPrinter.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::ISD::STACKSAVE
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1053
llvm::TargetLoweringBase::MaxStoresPerMemmove
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
Definition: TargetLowering.h:3461
llvm::ISD::ROTR
@ ROTR
Definition: ISDOpcodes.h:695
llvm::RegState::EarlyClobber
@ EarlyClobber
Register definition happens before uses.
Definition: MachineInstrBuilder.h:54
Debug.h
llvm::TargetLowering::getConstraintType
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
Definition: TargetLowering.cpp:5109
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:356
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:703
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:164
llvm::BPFSubtarget
Definition: BPFSubtarget.h:31
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:288
BPFTargetMachine.h
TargetLoweringObjectFileImpl.h