LLVM  13.0.0git
BPFISelLowering.cpp
Go to the documentation of this file.
1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that BPF uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "BPFISelLowering.h"
15 #include "BPF.h"
16 #include "BPFSubtarget.h"
17 #include "BPFTargetMachine.h"
25 #include "llvm/IR/DiagnosticInfo.h"
27 #include "llvm/Support/Debug.h"
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "bpf-lower"
33 
34 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
35  cl::Hidden, cl::init(false),
36  cl::desc("Expand memcpy into load/store pairs in order"));
37 
38 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
40  DAG.getContext()->diagnose(
41  DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
42 }
43 
44 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
45  SDValue Val) {
47  std::string Str;
48  raw_string_ostream OS(Str);
49  OS << Msg;
50  Val->print(OS);
51  OS.flush();
52  DAG.getContext()->diagnose(
53  DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc()));
54 }
55 
57  const BPFSubtarget &STI)
58  : TargetLowering(TM) {
59 
60  // Set up the register classes.
61  addRegisterClass(MVT::i64, &BPF::GPRRegClass);
62  if (STI.getHasAlu32())
63  addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
64 
65  // Compute derived properties from the register classes
67 
69 
74 
76 
80 
81  // Set unsupported atomic operations as Custom so
82  // we can emit better error messages than fatal error
83  // from selectiondag.
84  for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
85  if (VT == MVT::i32) {
86  if (STI.getHasAlu32())
87  continue;
88  } else {
90  }
91 
97  }
98 
99  for (auto VT : { MVT::i32, MVT::i64 }) {
100  if (VT == MVT::i32 && !STI.getHasAlu32())
101  continue;
102 
117 
121  }
122 
123  if (STI.getHasAlu32()) {
126  STI.getHasJmp32() ? Custom : Promote);
127  }
128 
133 
138 
139  // Extended load operations for i1 types must be promoted
140  for (MVT VT : MVT::integer_valuetypes()) {
144 
148  }
149 
151 
152  // Function alignments
155 
157  // LLVM generic code will try to expand memcpy into load/store pairs at this
158  // stage which is before quite a few IR optimization passes, therefore the
159  // loads and stores could potentially be moved apart from each other which
160  // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
161  // compilers.
162  //
163  // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
164  // of memcpy to later stage in IR optimization pipeline so those load/store
165  // pairs won't be touched and could be kept in order. Hence, we set
166  // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
167  // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
171  } else {
172  // inline memcpy() for kernel to see explicit copy
173  unsigned CommonMaxStores =
175 
176  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
177  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
179  }
180 
181  // CPU/Feature control
182  HasAlu32 = STI.getHasAlu32();
183  HasJmp32 = STI.getHasJmp32();
184  HasJmpExt = STI.getHasJmpExt();
185 }
186 
188  return false;
189 }
190 
191 bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
192  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
193  return false;
194  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
195  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
196  return NumBits1 > NumBits2;
197 }
198 
199 bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
200  if (!VT1.isInteger() || !VT2.isInteger())
201  return false;
202  unsigned NumBits1 = VT1.getSizeInBits();
203  unsigned NumBits2 = VT2.getSizeInBits();
204  return NumBits1 > NumBits2;
205 }
206 
207 bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
208  if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
209  return false;
210  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
211  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
212  return NumBits1 == 32 && NumBits2 == 64;
213 }
214 
215 bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
216  if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
217  return false;
218  unsigned NumBits1 = VT1.getSizeInBits();
219  unsigned NumBits2 = VT2.getSizeInBits();
220  return NumBits1 == 32 && NumBits2 == 64;
221 }
222 
223 std::pair<unsigned, const TargetRegisterClass *>
225  StringRef Constraint,
226  MVT VT) const {
227  if (Constraint.size() == 1)
228  // GCC Constraint Letters
229  switch (Constraint[0]) {
230  case 'r': // GENERAL_REGS
231  return std::make_pair(0U, &BPF::GPRRegClass);
232  default:
233  break;
234  }
235 
236  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
237 }
238 
239 void BPFTargetLowering::ReplaceNodeResults(
241  const char *err_msg;
242  uint32_t Opcode = N->getOpcode();
243  switch (Opcode) {
244  default:
245  report_fatal_error("Unhandled custom legalization");
248  case ISD::ATOMIC_LOAD_OR:
250  case ISD::ATOMIC_SWAP:
252  if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
253  err_msg = "Unsupported atomic operations, please use 32/64 bit version";
254  else
255  err_msg = "Unsupported atomic operations, please use 64 bit version";
256  break;
257  }
258 
259  SDLoc DL(N);
260  fail(DL, DAG, err_msg);
261 }
262 
264  switch (Op.getOpcode()) {
265  case ISD::BR_CC:
266  return LowerBR_CC(Op, DAG);
267  case ISD::GlobalAddress:
268  return LowerGlobalAddress(Op, DAG);
269  case ISD::SELECT_CC:
270  return LowerSELECT_CC(Op, DAG);
272  report_fatal_error("Unsupported dynamic stack allocation");
273  default:
274  llvm_unreachable("unimplemented operand");
275  }
276 }
277 
278 // Calling Convention Implementation
279 #include "BPFGenCallingConv.inc"
280 
281 SDValue BPFTargetLowering::LowerFormalArguments(
282  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
284  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
285  switch (CallConv) {
286  default:
287  report_fatal_error("Unsupported calling convention");
288  case CallingConv::C:
289  case CallingConv::Fast:
290  break;
291  }
292 
294  MachineRegisterInfo &RegInfo = MF.getRegInfo();
295 
296  // Assign locations to all of the incoming arguments.
298  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
299  CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
300 
301  for (auto &VA : ArgLocs) {
302  if (VA.isRegLoc()) {
303  // Arguments passed in registers
304  EVT RegVT = VA.getLocVT();
305  MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
306  switch (SimpleTy) {
307  default: {
308  errs() << "LowerFormalArguments Unhandled argument type: "
309  << RegVT.getEVTString() << '\n';
310  llvm_unreachable(0);
311  }
312  case MVT::i32:
313  case MVT::i64:
314  Register VReg = RegInfo.createVirtualRegister(
315  SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
316  RegInfo.addLiveIn(VA.getLocReg(), VReg);
317  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
318 
319  // If this is an value that has been promoted to wider types, insert an
320  // assert[sz]ext to capture this, then truncate to the right size.
321  if (VA.getLocInfo() == CCValAssign::SExt)
322  ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
323  DAG.getValueType(VA.getValVT()));
324  else if (VA.getLocInfo() == CCValAssign::ZExt)
325  ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
326  DAG.getValueType(VA.getValVT()));
327 
328  if (VA.getLocInfo() != CCValAssign::Full)
329  ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
330 
331  InVals.push_back(ArgValue);
332 
333  break;
334  }
335  } else {
336  fail(DL, DAG, "defined with too many args");
337  InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
338  }
339  }
340 
341  if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
342  fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
343  }
344 
345  return Chain;
346 }
347 
348 const unsigned BPFTargetLowering::MaxArgs = 5;
349 
350 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
351  SmallVectorImpl<SDValue> &InVals) const {
352  SelectionDAG &DAG = CLI.DAG;
353  auto &Outs = CLI.Outs;
354  auto &OutVals = CLI.OutVals;
355  auto &Ins = CLI.Ins;
356  SDValue Chain = CLI.Chain;
357  SDValue Callee = CLI.Callee;
358  bool &IsTailCall = CLI.IsTailCall;
359  CallingConv::ID CallConv = CLI.CallConv;
360  bool IsVarArg = CLI.IsVarArg;
362 
363  // BPF target does not support tail call optimization.
364  IsTailCall = false;
365 
366  switch (CallConv) {
367  default:
368  report_fatal_error("Unsupported calling convention");
369  case CallingConv::Fast:
370  case CallingConv::C:
371  break;
372  }
373 
374  // Analyze operands of the call, assigning locations to each operand.
376  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
377 
378  CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
379 
380  unsigned NumBytes = CCInfo.getNextStackOffset();
381 
382  if (Outs.size() > MaxArgs)
383  fail(CLI.DL, DAG, "too many args to ", Callee);
384 
385  for (auto &Arg : Outs) {
386  ISD::ArgFlagsTy Flags = Arg.Flags;
387  if (!Flags.isByVal())
388  continue;
389 
390  fail(CLI.DL, DAG, "pass by value not supported ", Callee);
391  }
392 
393  auto PtrVT = getPointerTy(MF.getDataLayout());
394  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
395 
396  SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
397 
398  // Walk arg assignments
399  for (unsigned i = 0,
400  e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
401  i != e; ++i) {
402  CCValAssign &VA = ArgLocs[i];
403  SDValue Arg = OutVals[i];
404 
405  // Promote the value if needed.
406  switch (VA.getLocInfo()) {
407  default:
408  llvm_unreachable("Unknown loc info");
409  case CCValAssign::Full:
410  break;
411  case CCValAssign::SExt:
412  Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
413  break;
414  case CCValAssign::ZExt:
415  Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
416  break;
417  case CCValAssign::AExt:
418  Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
419  break;
420  }
421 
422  // Push arguments into RegsToPass vector
423  if (VA.isRegLoc())
424  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
425  else
426  llvm_unreachable("call arg pass bug");
427  }
428 
429  SDValue InFlag;
430 
431  // Build a sequence of copy-to-reg nodes chained together with token chain and
432  // flag operands which copy the outgoing args into registers. The InFlag in
433  // necessary since all emitted instructions must be stuck together.
434  for (auto &Reg : RegsToPass) {
435  Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
436  InFlag = Chain.getValue(1);
437  }
438 
439  // If the callee is a GlobalAddress node (quite common, every direct call is)
440  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
441  // Likewise ExternalSymbol -> TargetExternalSymbol.
442  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
443  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
444  G->getOffset(), 0);
445  } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
446  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
447  fail(CLI.DL, DAG, Twine("A call to built-in function '"
448  + StringRef(E->getSymbol())
449  + "' is not supported."));
450  }
451 
452  // Returns a chain & a flag for retval copy to use.
453  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
455  Ops.push_back(Chain);
456  Ops.push_back(Callee);
457 
458  // Add argument registers to the end of the list so that they are
459  // known live into the call.
460  for (auto &Reg : RegsToPass)
461  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
462 
463  if (InFlag.getNode())
464  Ops.push_back(InFlag);
465 
466  Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
467  InFlag = Chain.getValue(1);
468 
469  // Create the CALLSEQ_END node.
470  Chain = DAG.getCALLSEQ_END(
471  Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true),
472  DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL);
473  InFlag = Chain.getValue(1);
474 
475  // Handle result values, copying them out of physregs into vregs that we
476  // return.
477  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
478  InVals);
479 }
480 
481 SDValue
482 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
483  bool IsVarArg,
485  const SmallVectorImpl<SDValue> &OutVals,
486  const SDLoc &DL, SelectionDAG &DAG) const {
487  unsigned Opc = BPFISD::RET_FLAG;
488 
489  // CCValAssign - represent the assignment of the return value to a location
492 
493  // CCState - Info about the registers and stack slot.
494  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
495 
496  if (MF.getFunction().getReturnType()->isAggregateType()) {
497  fail(DL, DAG, "only integer returns supported");
498  return DAG.getNode(Opc, DL, MVT::Other, Chain);
499  }
500 
501  // Analize return values.
502  CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
503 
504  SDValue Flag;
505  SmallVector<SDValue, 4> RetOps(1, Chain);
506 
507  // Copy the result values into the output registers.
508  for (unsigned i = 0; i != RVLocs.size(); ++i) {
509  CCValAssign &VA = RVLocs[i];
510  assert(VA.isRegLoc() && "Can only return in registers!");
511 
512  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
513 
514  // Guarantee that all emitted copies are stuck together,
515  // avoiding something bad.
516  Flag = Chain.getValue(1);
517  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
518  }
519 
520  RetOps[0] = Chain; // Update chain.
521 
522  // Add the flag if we have it.
523  if (Flag.getNode())
524  RetOps.push_back(Flag);
525 
526  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
527 }
528 
529 SDValue BPFTargetLowering::LowerCallResult(
530  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
532  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
533 
535  // Assign locations to each value returned by this call.
537  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
538 
539  if (Ins.size() >= 2) {
540  fail(DL, DAG, "only small returns supported");
541  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
542  InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
543  return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
544  }
545 
546  CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
547 
548  // Copy all of the result registers out of their specified physreg.
549  for (auto &Val : RVLocs) {
550  Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
551  Val.getValVT(), InFlag).getValue(1);
552  InFlag = Chain.getValue(2);
553  InVals.push_back(Chain.getValue(0));
554  }
555 
556  return Chain;
557 }
558 
559 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
560  switch (CC) {
561  default:
562  break;
563  case ISD::SETULT:
564  case ISD::SETULE:
565  case ISD::SETLT:
566  case ISD::SETLE:
568  std::swap(LHS, RHS);
569  break;
570  }
571 }
572 
573 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
574  SDValue Chain = Op.getOperand(0);
575  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
576  SDValue LHS = Op.getOperand(2);
577  SDValue RHS = Op.getOperand(3);
578  SDValue Dest = Op.getOperand(4);
579  SDLoc DL(Op);
580 
581  if (!getHasJmpExt())
582  NegateCC(LHS, RHS, CC);
583 
584  return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
585  DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
586 }
587 
588 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
589  SDValue LHS = Op.getOperand(0);
590  SDValue RHS = Op.getOperand(1);
591  SDValue TrueV = Op.getOperand(2);
592  SDValue FalseV = Op.getOperand(3);
593  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
594  SDLoc DL(Op);
595 
596  if (!getHasJmpExt())
597  NegateCC(LHS, RHS, CC);
598 
599  SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
600  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
601  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
602 
603  return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
604 }
605 
606 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
607  switch ((BPFISD::NodeType)Opcode) {
609  break;
610  case BPFISD::RET_FLAG:
611  return "BPFISD::RET_FLAG";
612  case BPFISD::CALL:
613  return "BPFISD::CALL";
614  case BPFISD::SELECT_CC:
615  return "BPFISD::SELECT_CC";
616  case BPFISD::BR_CC:
617  return "BPFISD::BR_CC";
618  case BPFISD::Wrapper:
619  return "BPFISD::Wrapper";
620  case BPFISD::MEMCPY:
621  return "BPFISD::MEMCPY";
622  }
623  return nullptr;
624 }
625 
626 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
627  SelectionDAG &DAG) const {
628  auto N = cast<GlobalAddressSDNode>(Op);
629  assert(N->getOffset() == 0 && "Invalid offset for global address");
630 
631  SDLoc DL(Op);
632  const GlobalValue *GV = N->getGlobal();
633  SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
634 
635  return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
636 }
637 
638 unsigned
639 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
640  unsigned Reg, bool isSigned) const {
641  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
643  int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
644  MachineFunction *F = BB->getParent();
645  DebugLoc DL = MI.getDebugLoc();
646 
647  MachineRegisterInfo &RegInfo = F->getRegInfo();
648 
649  if (!isSigned) {
650  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
651  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
652  return PromotedReg0;
653  }
654  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
655  Register PromotedReg1 = RegInfo.createVirtualRegister(RC);
656  Register PromotedReg2 = RegInfo.createVirtualRegister(RC);
657  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
658  BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
659  .addReg(PromotedReg0).addImm(32);
660  BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
661  .addReg(PromotedReg1).addImm(32);
662 
663  return PromotedReg2;
664 }
665 
667 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
669  const {
670  MachineFunction *MF = MI.getParent()->getParent();
672  MachineInstrBuilder MIB(*MF, MI);
673  unsigned ScratchReg;
674 
675  // This function does custom insertion during lowering BPFISD::MEMCPY which
676  // only has two register operands from memcpy semantics, the copy source
677  // address and the copy destination address.
678  //
679  // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
680  // a third scratch register to serve as the destination register of load and
681  // source register of store.
682  //
683  // The scratch register here is with the Define | Dead | EarlyClobber flags.
684  // The EarlyClobber flag has the semantic property that the operand it is
685  // attached to is clobbered before the rest of the inputs are read. Hence it
686  // must be unique among the operands to the instruction. The Define flag is
687  // needed to coerce the machine verifier that an Undef value isn't a problem
688  // as we anyway is loading memory into it. The Dead flag is needed as the
689  // value in scratch isn't supposed to be used by any other instruction.
690  ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
691  MIB.addReg(ScratchReg,
693 
694  return BB;
695 }
696 
699  MachineBasicBlock *BB) const {
700  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
701  DebugLoc DL = MI.getDebugLoc();
702  unsigned Opc = MI.getOpcode();
703  bool isSelectRROp = (Opc == BPF::Select ||
704  Opc == BPF::Select_64_32 ||
705  Opc == BPF::Select_32 ||
706  Opc == BPF::Select_32_64);
707 
708  bool isMemcpyOp = Opc == BPF::MEMCPY;
709 
710 #ifndef NDEBUG
711  bool isSelectRIOp = (Opc == BPF::Select_Ri ||
712  Opc == BPF::Select_Ri_64_32 ||
713  Opc == BPF::Select_Ri_32 ||
714  Opc == BPF::Select_Ri_32_64);
715 
716 
717  assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
718  "Unexpected instr type to insert");
719 #endif
720 
721  if (isMemcpyOp)
722  return EmitInstrWithCustomInserterMemcpy(MI, BB);
723 
724  bool is32BitCmp = (Opc == BPF::Select_32 ||
725  Opc == BPF::Select_32_64 ||
726  Opc == BPF::Select_Ri_32 ||
727  Opc == BPF::Select_Ri_32_64);
728 
729  // To "insert" a SELECT instruction, we actually have to insert the diamond
730  // control-flow pattern. The incoming instruction knows the destination vreg
731  // to set, the condition code register to branch on, the true/false values to
732  // select between, and a branch opcode to use.
733  const BasicBlock *LLVM_BB = BB->getBasicBlock();
734  MachineFunction::iterator I = ++BB->getIterator();
735 
736  // ThisMBB:
737  // ...
738  // TrueVal = ...
739  // jmp_XX r1, r2 goto Copy1MBB
740  // fallthrough --> Copy0MBB
741  MachineBasicBlock *ThisMBB = BB;
742  MachineFunction *F = BB->getParent();
743  MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
744  MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
745 
746  F->insert(I, Copy0MBB);
747  F->insert(I, Copy1MBB);
748  // Update machine-CFG edges by transferring all successors of the current
749  // block to the new block which will contain the Phi node for the select.
750  Copy1MBB->splice(Copy1MBB->begin(), BB,
751  std::next(MachineBasicBlock::iterator(MI)), BB->end());
753  // Next, add the true and fallthrough blocks as its successors.
754  BB->addSuccessor(Copy0MBB);
755  BB->addSuccessor(Copy1MBB);
756 
757  // Insert Branch if Flag
758  int CC = MI.getOperand(3).getImm();
759  int NewCC;
760  switch (CC) {
761 #define SET_NEWCC(X, Y) \
762  case ISD::X: \
763  if (is32BitCmp && HasJmp32) \
764  NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
765  else \
766  NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
767  break
768  SET_NEWCC(SETGT, JSGT);
769  SET_NEWCC(SETUGT, JUGT);
770  SET_NEWCC(SETGE, JSGE);
771  SET_NEWCC(SETUGE, JUGE);
772  SET_NEWCC(SETEQ, JEQ);
773  SET_NEWCC(SETNE, JNE);
774  SET_NEWCC(SETLT, JSLT);
775  SET_NEWCC(SETULT, JULT);
776  SET_NEWCC(SETLE, JSLE);
777  SET_NEWCC(SETULE, JULE);
778  default:
779  report_fatal_error("unimplemented select CondCode " + Twine(CC));
780  }
781 
782  Register LHS = MI.getOperand(1).getReg();
783  bool isSignedCmp = (CC == ISD::SETGT ||
784  CC == ISD::SETGE ||
785  CC == ISD::SETLT ||
786  CC == ISD::SETLE);
787 
788  // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
789  // to be promoted, however if the 32-bit comparison operands are destination
790  // registers then they are implicitly zero-extended already, there is no
791  // need of explicit zero-extend sequence for them.
792  //
793  // We simply do extension for all situations in this method, but we will
794  // try to remove those unnecessary in BPFMIPeephole pass.
795  if (is32BitCmp && !HasJmp32)
796  LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
797 
798  if (isSelectRROp) {
799  Register RHS = MI.getOperand(2).getReg();
800 
801  if (is32BitCmp && !HasJmp32)
802  RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
803 
804  BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
805  } else {
806  int64_t imm32 = MI.getOperand(2).getImm();
807  // sanity check before we build J*_ri instruction.
808  assert (isInt<32>(imm32));
809  BuildMI(BB, DL, TII.get(NewCC))
810  .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
811  }
812 
813  // Copy0MBB:
814  // %FalseValue = ...
815  // # fallthrough to Copy1MBB
816  BB = Copy0MBB;
817 
818  // Update machine-CFG edges
819  BB->addSuccessor(Copy1MBB);
820 
821  // Copy1MBB:
822  // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
823  // ...
824  BB = Copy1MBB;
825  BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
826  .addReg(MI.getOperand(5).getReg())
827  .addMBB(Copy0MBB)
828  .addReg(MI.getOperand(4).getReg())
829  .addMBB(ThisMBB);
830 
831  MI.eraseFromParent(); // The pseudo instruction is gone now.
832  return BB;
833 }
834 
836  EVT VT) const {
837  return getHasAlu32() ? MVT::i32 : MVT::i64;
838 }
839 
841  EVT VT) const {
842  return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
843 }
llvm::CCValAssign::getLocVT
MVT getLocVT() const
Definition: CallingConvLower.h:153
llvm::MachineRegisterInfo::addLiveIn
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Definition: MachineRegisterInfo.h:929
i
i
Definition: README.txt:29
llvm::ISD::SETUGE
@ SETUGE
Definition: ISDOpcodes.h:1354
llvm::TargetLoweringBase::MaxStoresPerMemsetOptSize
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3066
llvm::CCValAssign::ZExt
@ ZExt
Definition: CallingConvLower.h:38
ValueTypes.h
llvm::BPFTargetLowering::getTargetNodeName
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Definition: BPFISelLowering.cpp:606
llvm::SelectionDAG::getCALLSEQ_START
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:919
llvm::ISD::SETLE
@ SETLE
Definition: ISDOpcodes.h:1365
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
Definition: AllocatorList.h:23
BPFSubtarget.h
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::SystemZISD::TM
@ TM
Definition: SystemZISelLowering.h:65
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1078
llvm::CCValAssign::Full
@ Full
Definition: CallingConvLower.h:36
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::ISD::SETGT
@ SETGT
Definition: ISDOpcodes.h:1362
llvm::ISD::SETNE
@ SETNE
Definition: ISDOpcodes.h:1366
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:158
llvm::ISD::BR_JT
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:936
llvm::DiagnosticInfoUnsupported
Diagnostic information for unsupported feature in backend.
Definition: DiagnosticInfo.h:993
llvm::BPFSubtarget::getHasJmp32
bool getHasJmp32() const
Definition: BPFSubtarget.h:72
llvm::CCState
CCState - This class holds information needed while lowering arguments and return values.
Definition: CallingConvLower.h:191
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:735
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:59
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:152
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:656
llvm::ISD::DYNAMIC_STACKALLOC
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:921
llvm::BPFISD::MEMCPY
@ MEMCPY
Definition: BPFISelLowering.h:31
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:664
llvm::SelectionDAG::getValueType
SDValue getValueType(EVT)
Definition: SelectionDAG.cpp:1697
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:614
llvm::BPFSubtarget::getRegisterInfo
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:86
llvm::TargetLoweringBase::MaxStoresPerMemset
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
Definition: TargetLowering.h:3064
BPFISelLowering.h
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::ISD::SETEQ
@ SETEQ
Definition: ISDOpcodes.h:1361
llvm::ISD::STACKRESTORE
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1002
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:8053
llvm::BPFTargetLowering::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFISelLowering.h:59
llvm::BPFTargetLowering::getHasAlu32
bool getHasAlu32() const
Definition: BPFISelLowering.h:57
ErrorHandling.h
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:722
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:455
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:249
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:231
llvm::ISD::SETULE
@ SETULE
Definition: ISDOpcodes.h:1356
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::ISD::SHL_PARTS
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:705
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:692
llvm::MVT::integer_valuetypes
static mvt_range integer_valuetypes()
Definition: MachineValueType.h:1372
llvm::ARMISD::MEMCPY
@ MEMCPY
Definition: ARMISelLowering.h:301
llvm::TargetLoweringBase::setMinFunctionAlignment
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
Definition: TargetLowering.h:2261
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:851
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:892
llvm::ISD::EXTLOAD
@ EXTLOAD
Definition: ISDOpcodes.h:1321
BPF.h
llvm::BPFISD::NodeType
NodeType
Definition: BPFISelLowering.h:24
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:447
llvm::ISD::ATOMIC_LOAD_OR
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1130
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::ISD::BRCOND
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:946
MachineRegisterInfo.h
llvm::BPFTargetLowering::getRegForInlineAsmConstraint
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
Definition: BPFISelLowering.cpp:224
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:1953
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:32
llvm::ISD::BRIND
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:932
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:650
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:205
llvm::BPFISD::BR_CC
@ BR_CC
Definition: BPFISelLowering.h:29
llvm::BPFISD::FIRST_NUMBER
@ FIRST_NUMBER
Definition: BPFISelLowering.h:25
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:953
llvm::BPFTargetLowering::getSetCCResultType
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
Definition: BPFISelLowering.cpp:835
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:40
llvm::TargetLowering::CallLoweringInfo::IsVarArg
bool IsVarArg
Definition: TargetLowering.h:3699
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:568
llvm::CCValAssign::AExt
@ AExt
Definition: CallingConvLower.h:39
llvm::ISD::GlobalAddress
@ GlobalAddress
Definition: ISDOpcodes.h:71
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:684
llvm::ExternalSymbolSDNode
Definition: SelectionDAGNodes.h:2144
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1113
llvm::CCValAssign
CCValAssign - Represent assignment of one arg/retval to a location.
Definition: CallingConvLower.h:33
llvm::BPFISD::SELECT_CC
@ SELECT_CC
Definition: BPFISelLowering.h:28
llvm::ISD::CTLZ
@ CTLZ
Definition: ISDOpcodes.h:658
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:216
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:669
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:719
llvm::ISD::ArgFlagsTy::isByVal
bool isByVal() const
Definition: TargetCallingConv.h:83
llvm::ISD::SETGE
@ SETGE
Definition: ISDOpcodes.h:1363
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::CCValAssign::getLocReg
Register getLocReg() const
Definition: CallingConvLower.h:150
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:737
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1347
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3143
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:725
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:304
llvm::BPFTargetLowering::BPFTargetLowering
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
Definition: BPFISelLowering.cpp:56
llvm::ISD::UDIVREM
@ UDIVREM
Definition: ISDOpcodes.h:249
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::TargetLoweringBase::addRegisterClass
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
Definition: TargetLowering.h:2141
llvm::ISD::ATOMIC_LOAD_AND
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1128
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:139
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:146
llvm::BPFTargetLowering::EmitInstrWithCustomInserter
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: BPFISelLowering.cpp:698
llvm::report_fatal_error
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::raw_ostream::flush
void flush()
Definition: raw_ostream.h:183
llvm::RegState::EarlyClobber
@ EarlyClobber
Register definition happens before uses.
Definition: MachineInstrBuilder.h:54
llvm::TargetLoweringBase::MaxStoresPerMemcpy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
Definition: TargetLowering.h:3079
llvm::CCValAssign::getLocInfo
LocInfo getLocInfo() const
Definition: CallingConvLower.h:155
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::CCValAssign::isRegLoc
bool isRegLoc() const
Definition: CallingConvLower.h:145
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::ISD::SETUGT
@ SETUGT
Definition: ISDOpcodes.h:1353
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:683
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:202
llvm::TargetLowering::CallLoweringInfo::Outs
SmallVector< ISD::OutputArg, 32 > Outs
Definition: TargetLowering.h:3722
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
llvm::TargetLoweringBase::setOperationAction
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
Definition: TargetLowering.h:2158
llvm::TargetLoweringBase::MaxStoresPerMemcpyOptSize
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3081
llvm::cl::opt< bool >
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::isInt< 32 >
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:373
llvm::BPFTargetLowering::LowerOperation
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Definition: BPFISelLowering.cpp:263
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:333
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:180
llvm::CCValAssign::SExt
@ SExt
Definition: CallingConvLower.h:37
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1117
llvm::TargetLowering::CallLoweringInfo::Chain
SDValue Chain
Definition: TargetLowering.h:3695
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:60
llvm::TargetLoweringBase::Promote
@ Promote
Definition: TargetLowering.h:195
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:761
llvm::TargetLowering::CallLoweringInfo::CallConv
CallingConv::ID CallConv
Definition: TargetLowering.h:3716
llvm::TargetLoweringBase::setStackPointerRegisterToSaveRestore
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
Definition: TargetLowering.h:2107
fail
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
Definition: BPFISelLowering.cpp:38
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:665
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:7818
llvm::TargetLoweringBase::setPrefFunctionAlignment
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
Definition: TargetLowering.h:2267
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::ISD::ATOMIC_LOAD_ADD
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1126
llvm::TargetLowering::CallLoweringInfo::DL
SDLoc DL
Definition: TargetLowering.h:3720
llvm::Function::hasStructRetAttr
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:694
llvm::ISD::ZEXTLOAD
@ ZEXTLOAD
Definition: ISDOpcodes.h:1321
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:172
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:41
llvm::TargetLowering::CallLoweringInfo
This structure contains all information that is necessary for lowering calls.
Definition: TargetLowering.h:3694
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::BPFSubtarget::getSelectionDAGInfo
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:83
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::ISD::MULHS
@ MULHS
Definition: ISDOpcodes.h:612
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:39
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::ISD::SETULT
@ SETULT
Definition: ISDOpcodes.h:1355
llvm::SDNode::print
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
Definition: SelectionDAGDumper.cpp:1047
BPFExpandMemcpyInOrder
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1341
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:30
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::ISD::SRA_PARTS
@ SRA_PARTS
Definition: ISDOpcodes.h:706
NegateCC
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
Definition: BPFISelLowering.cpp:559
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::SelectionDAG::getCALLSEQ_END
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
Definition: SelectionDAG.h:931
llvm::TargetLowering::CallLoweringInfo::Ins
SmallVector< ISD::InputArg, 32 > Ins
Definition: TargetLowering.h:3724
llvm::TargetLowering::CallLoweringInfo::DAG
SelectionDAG & DAG
Definition: TargetLowering.h:3719
llvm::BPFSelectionDAGInfo::getCommonMaxStoresPerMemFunc
unsigned getCommonMaxStoresPerMemFunc() const
Definition: BPFSelectionDAGInfo.h:29
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:44
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:863
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::ISD::SREM
@ SREM
Definition: ISDOpcodes.h:237
llvm::ISD::UMUL_LOHI
@ UMUL_LOHI
Definition: ISDOpcodes.h:244
uint32_t
llvm::ISD::ArgFlagsTy
Definition: TargetCallingConv.h:27
llvm::TargetLoweringBase::MaxStoresPerMemmoveOptSize
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3116
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::BPFISD::CALL
@ CALL
Definition: BPFISelLowering.h:27
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:243
llvm::TargetLowering::getRegForInlineAsmConstraint
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Definition: TargetLowering.cpp:4550
llvm::CallingConv::C
@ C
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1125
llvm::ISD::SEXTLOAD
@ SEXTLOAD
Definition: ISDOpcodes.h:1321
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:205
SET_NEWCC
#define SET_NEWCC(X, Y)
llvm::EVT::getEVTString
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition: ValueTypes.cpp:152
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:162
llvm::TargetLowering::CallLoweringInfo::IsTailCall
bool IsTailCall
Definition: TargetLowering.h:3710
llvm::ISD::SETLT
@ SETLT
Definition: ISDOpcodes.h:1364
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:228
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
llvm::BPFTargetLowering::getScalarShiftAmountTy
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
Definition: BPFISelLowering.cpp:840
llvm::TargetLoweringBase::setLoadExtAction
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
Definition: TargetLowering.h:2166
llvm::GlobalAddressSDNode
Definition: SelectionDAGNodes.h:1688
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:524
CallingConvLower.h
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:314
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::ISD::getSetCCSwappedOperands
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
Definition: SelectionDAG.cpp:453
DiagnosticInfo.h
llvm::BPFSubtarget::getHasAlu32
bool getHasAlu32() const
Definition: BPFSubtarget.h:73
llvm::TargetLoweringBase::Custom
@ Custom
Definition: TargetLowering.h:198
llvm::SelectionDAG::getTargetExternalSymbol
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.cpp:1728
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:43
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:138
llvm::TargetLoweringBase::ZeroOrOneBooleanContent
@ ZeroOrOneBooleanContent
Definition: TargetLowering.h:230
llvm::ISD::SRL_PARTS
@ SRL_PARTS
Definition: ISDOpcodes.h:707
llvm::BPFISD::Wrapper
@ Wrapper
Definition: BPFISelLowering.h:30
llvm::TargetLoweringBase::setBooleanContents
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
Definition: TargetLowering.h:2075
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
MachineInstrBuilder.h
llvm::ISD::UREM
@ UREM
Definition: ISDOpcodes.h:238
llvm::TargetLoweringBase::Expand
@ Expand
Definition: TargetLowering.h:196
llvm::BPFTargetLowering::isOffsetFoldingLegal
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
Definition: BPFISelLowering.cpp:187
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
N
#define N
llvm::TargetLoweringBase::computeRegisterProperties
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
Definition: TargetLoweringBase.cpp:1255
llvm::ISD::CTTZ
@ CTTZ
Definition: ISDOpcodes.h:657
llvm::TargetLoweringBase::getRegClassFor
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
Definition: TargetLowering.h:839
llvm::MachineFunction::getDataLayout
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Definition: MachineFunction.cpp:260
llvm::MipsISD::Ins
@ Ins
Definition: MipsISelLowering.h:157
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:611
llvm::TargetLowering::CallLoweringInfo::OutVals
SmallVector< SDValue, 32 > OutVals
Definition: TargetLowering.h:3723
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:884
llvm::TargetLowering::CallLoweringInfo::Callee
SDValue Callee
Definition: TargetLowering.h:3717
llvm::BPFSubtarget::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFSubtarget.h:71
llvm::MVT::i16
@ i16
Definition: MachineValueType.h:42
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:437
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:265
llvm::BPFISD::RET_FLAG
@ RET_FLAG
Definition: BPFISelLowering.h:26
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:248
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:414
llvm::ISD::ATOMIC_LOAD_XOR
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1131
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:716
raw_ostream.h
llvm::StringRef::size
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:160
MachineFunction.h
DiagnosticPrinter.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::ISD::STACKSAVE
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:998
llvm::TargetLoweringBase::MaxStoresPerMemmove
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
Definition: TargetLowering.h:3114
llvm::ISD::ROTR
@ ROTR
Definition: ISDOpcodes.h:651
Debug.h
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:343
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:659
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:122
llvm::BPFSubtarget
Definition: BPFSubtarget.h:31
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:281
BPFTargetMachine.h
TargetLoweringObjectFileImpl.h