LLVM  9.0.0svn
BPFISelLowering.cpp
Go to the documentation of this file.
1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that BPF uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "BPFISelLowering.h"
15 #include "BPF.h"
16 #include "BPFSubtarget.h"
17 #include "BPFTargetMachine.h"
26 #include "llvm/IR/DiagnosticInfo.h"
28 #include "llvm/Support/Debug.h"
31 using namespace llvm;
32 
33 #define DEBUG_TYPE "bpf-lower"
34 
35 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
36  cl::Hidden, cl::init(false),
37  cl::desc("Expand memcpy into load/store pairs in order"));
38 
39 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
41  DAG.getContext()->diagnose(
43 }
44 
45 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
46  SDValue Val) {
48  std::string Str;
49  raw_string_ostream OS(Str);
50  OS << Msg;
51  Val->print(OS);
52  OS.flush();
53  DAG.getContext()->diagnose(
55 }
56 
58  const BPFSubtarget &STI)
59  : TargetLowering(TM) {
60 
61  // Set up the register classes.
62  addRegisterClass(MVT::i64, &BPF::GPRRegClass);
63  if (STI.getHasAlu32())
64  addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
65 
66  // Compute derived properties from the register classes
68 
70 
75 
77 
81 
82  for (auto VT : { MVT::i32, MVT::i64 }) {
83  if (VT == MVT::i32 && !STI.getHasAlu32())
84  continue;
85 
100 
104  }
105 
106  if (STI.getHasAlu32()) {
109  STI.getHasJmp32() ? Custom : Promote);
110  }
111 
116 
121 
122  // Extended load operations for i1 types must be promoted
123  for (MVT VT : MVT::integer_valuetypes()) {
127 
131  }
132 
134 
135  // Function alignments (log2)
138 
140  // LLVM generic code will try to expand memcpy into load/store pairs at this
141  // stage which is before quite a few IR optimization passes, therefore the
142  // loads and stores could potentially be moved apart from each other which
143  // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
144  // compilers.
145  //
146  // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
147  // of memcpy to later stage in IR optimization pipeline so those load/store
148  // pairs won't be touched and could be kept in order. Hence, we set
149  // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
150  // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
154  } else {
155  // inline memcpy() for kernel to see explicit copy
156  unsigned CommonMaxStores =
158 
159  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
160  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
162  }
163 
164  // CPU/Feature control
165  HasAlu32 = STI.getHasAlu32();
166  HasJmp32 = STI.getHasJmp32();
167  HasJmpExt = STI.getHasJmpExt();
168 }
169 
171  return false;
172 }
173 
174 std::pair<unsigned, const TargetRegisterClass *>
176  StringRef Constraint,
177  MVT VT) const {
178  if (Constraint.size() == 1)
179  // GCC Constraint Letters
180  switch (Constraint[0]) {
181  case 'r': // GENERAL_REGS
182  return std::make_pair(0U, &BPF::GPRRegClass);
183  default:
184  break;
185  }
186 
187  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
188 }
189 
191  switch (Op.getOpcode()) {
192  case ISD::BR_CC:
193  return LowerBR_CC(Op, DAG);
194  case ISD::GlobalAddress:
195  return LowerGlobalAddress(Op, DAG);
196  case ISD::SELECT_CC:
197  return LowerSELECT_CC(Op, DAG);
198  default:
199  llvm_unreachable("unimplemented operand");
200  }
201 }
202 
203 // Calling Convention Implementation
204 #include "BPFGenCallingConv.inc"
205 
206 SDValue BPFTargetLowering::LowerFormalArguments(
207  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
208  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
209  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
210  switch (CallConv) {
211  default:
212  report_fatal_error("Unsupported calling convention");
213  case CallingConv::C:
214  case CallingConv::Fast:
215  break;
216  }
217 
219  MachineRegisterInfo &RegInfo = MF.getRegInfo();
220 
221  // Assign locations to all of the incoming arguments.
223  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
224  CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
225 
226  for (auto &VA : ArgLocs) {
227  if (VA.isRegLoc()) {
228  // Arguments passed in registers
229  EVT RegVT = VA.getLocVT();
230  MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
231  switch (SimpleTy) {
232  default: {
233  errs() << "LowerFormalArguments Unhandled argument type: "
234  << RegVT.getEVTString() << '\n';
235  llvm_unreachable(0);
236  }
237  case MVT::i32:
238  case MVT::i64:
239  unsigned VReg = RegInfo.createVirtualRegister(SimpleTy == MVT::i64 ?
240  &BPF::GPRRegClass :
241  &BPF::GPR32RegClass);
242  RegInfo.addLiveIn(VA.getLocReg(), VReg);
243  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
244 
245  // If this is an value that has been promoted to wider types, insert an
246  // assert[sz]ext to capture this, then truncate to the right size.
247  if (VA.getLocInfo() == CCValAssign::SExt)
248  ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
249  DAG.getValueType(VA.getValVT()));
250  else if (VA.getLocInfo() == CCValAssign::ZExt)
251  ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
252  DAG.getValueType(VA.getValVT()));
253 
254  if (VA.getLocInfo() != CCValAssign::Full)
255  ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
256 
257  InVals.push_back(ArgValue);
258 
259  break;
260  }
261  } else {
262  fail(DL, DAG, "defined with too many args");
263  InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
264  }
265  }
266 
267  if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
268  fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
269  }
270 
271  return Chain;
272 }
273 
274 const unsigned BPFTargetLowering::MaxArgs = 5;
275 
276 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
277  SmallVectorImpl<SDValue> &InVals) const {
278  SelectionDAG &DAG = CLI.DAG;
279  auto &Outs = CLI.Outs;
280  auto &OutVals = CLI.OutVals;
281  auto &Ins = CLI.Ins;
282  SDValue Chain = CLI.Chain;
283  SDValue Callee = CLI.Callee;
284  bool &IsTailCall = CLI.IsTailCall;
285  CallingConv::ID CallConv = CLI.CallConv;
286  bool IsVarArg = CLI.IsVarArg;
288 
289  // BPF target does not support tail call optimization.
290  IsTailCall = false;
291 
292  switch (CallConv) {
293  default:
294  report_fatal_error("Unsupported calling convention");
295  case CallingConv::Fast:
296  case CallingConv::C:
297  break;
298  }
299 
300  // Analyze operands of the call, assigning locations to each operand.
302  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
303 
304  CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
305 
306  unsigned NumBytes = CCInfo.getNextStackOffset();
307 
308  if (Outs.size() > MaxArgs)
309  fail(CLI.DL, DAG, "too many args to ", Callee);
310 
311  for (auto &Arg : Outs) {
312  ISD::ArgFlagsTy Flags = Arg.Flags;
313  if (!Flags.isByVal())
314  continue;
315 
316  fail(CLI.DL, DAG, "pass by value not supported ", Callee);
317  }
318 
319  auto PtrVT = getPointerTy(MF.getDataLayout());
320  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
321 
322  SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
323 
324  // Walk arg assignments
325  for (unsigned i = 0,
326  e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
327  i != e; ++i) {
328  CCValAssign &VA = ArgLocs[i];
329  SDValue Arg = OutVals[i];
330 
331  // Promote the value if needed.
332  switch (VA.getLocInfo()) {
333  default:
334  llvm_unreachable("Unknown loc info");
335  case CCValAssign::Full:
336  break;
337  case CCValAssign::SExt:
338  Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
339  break;
340  case CCValAssign::ZExt:
341  Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
342  break;
343  case CCValAssign::AExt:
344  Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
345  break;
346  }
347 
348  // Push arguments into RegsToPass vector
349  if (VA.isRegLoc())
350  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
351  else
352  llvm_unreachable("call arg pass bug");
353  }
354 
355  SDValue InFlag;
356 
357  // Build a sequence of copy-to-reg nodes chained together with token chain and
358  // flag operands which copy the outgoing args into registers. The InFlag in
359  // necessary since all emitted instructions must be stuck together.
360  for (auto &Reg : RegsToPass) {
361  Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
362  InFlag = Chain.getValue(1);
363  }
364 
365  // If the callee is a GlobalAddress node (quite common, every direct call is)
366  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
367  // Likewise ExternalSymbol -> TargetExternalSymbol.
368  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
369  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
370  G->getOffset(), 0);
371  } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
372  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
373  fail(CLI.DL, DAG, Twine("A call to built-in function '"
374  + StringRef(E->getSymbol())
375  + "' is not supported."));
376  }
377 
378  // Returns a chain & a flag for retval copy to use.
379  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
381  Ops.push_back(Chain);
382  Ops.push_back(Callee);
383 
384  // Add argument registers to the end of the list so that they are
385  // known live into the call.
386  for (auto &Reg : RegsToPass)
387  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
388 
389  if (InFlag.getNode())
390  Ops.push_back(InFlag);
391 
392  Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
393  InFlag = Chain.getValue(1);
394 
395  // Create the CALLSEQ_END node.
396  Chain = DAG.getCALLSEQ_END(
397  Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true),
398  DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL);
399  InFlag = Chain.getValue(1);
400 
401  // Handle result values, copying them out of physregs into vregs that we
402  // return.
403  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
404  InVals);
405 }
406 
407 SDValue
408 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
409  bool IsVarArg,
411  const SmallVectorImpl<SDValue> &OutVals,
412  const SDLoc &DL, SelectionDAG &DAG) const {
413  unsigned Opc = BPFISD::RET_FLAG;
414 
415  // CCValAssign - represent the assignment of the return value to a location
418 
419  // CCState - Info about the registers and stack slot.
420  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
421 
422  if (MF.getFunction().getReturnType()->isAggregateType()) {
423  fail(DL, DAG, "only integer returns supported");
424  return DAG.getNode(Opc, DL, MVT::Other, Chain);
425  }
426 
427  // Analize return values.
428  CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
429 
430  SDValue Flag;
431  SmallVector<SDValue, 4> RetOps(1, Chain);
432 
433  // Copy the result values into the output registers.
434  for (unsigned i = 0; i != RVLocs.size(); ++i) {
435  CCValAssign &VA = RVLocs[i];
436  assert(VA.isRegLoc() && "Can only return in registers!");
437 
438  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
439 
440  // Guarantee that all emitted copies are stuck together,
441  // avoiding something bad.
442  Flag = Chain.getValue(1);
443  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
444  }
445 
446  RetOps[0] = Chain; // Update chain.
447 
448  // Add the flag if we have it.
449  if (Flag.getNode())
450  RetOps.push_back(Flag);
451 
452  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
453 }
454 
455 SDValue BPFTargetLowering::LowerCallResult(
456  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
457  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
458  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
459 
461  // Assign locations to each value returned by this call.
463  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
464 
465  if (Ins.size() >= 2) {
466  fail(DL, DAG, "only small returns supported");
467  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
468  InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
469  return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
470  }
471 
472  CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
473 
474  // Copy all of the result registers out of their specified physreg.
475  for (auto &Val : RVLocs) {
476  Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
477  Val.getValVT(), InFlag).getValue(1);
478  InFlag = Chain.getValue(2);
479  InVals.push_back(Chain.getValue(0));
480  }
481 
482  return Chain;
483 }
484 
485 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
486  switch (CC) {
487  default:
488  break;
489  case ISD::SETULT:
490  case ISD::SETULE:
491  case ISD::SETLT:
492  case ISD::SETLE:
494  std::swap(LHS, RHS);
495  break;
496  }
497 }
498 
499 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
500  SDValue Chain = Op.getOperand(0);
501  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
502  SDValue LHS = Op.getOperand(2);
503  SDValue RHS = Op.getOperand(3);
504  SDValue Dest = Op.getOperand(4);
505  SDLoc DL(Op);
506 
507  if (!getHasJmpExt())
508  NegateCC(LHS, RHS, CC);
509 
510  return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
511  DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
512 }
513 
514 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
515  SDValue LHS = Op.getOperand(0);
516  SDValue RHS = Op.getOperand(1);
517  SDValue TrueV = Op.getOperand(2);
518  SDValue FalseV = Op.getOperand(3);
519  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
520  SDLoc DL(Op);
521 
522  if (!getHasJmpExt())
523  NegateCC(LHS, RHS, CC);
524 
525  SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
526  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
527  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
528 
529  return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
530 }
531 
532 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
533  switch ((BPFISD::NodeType)Opcode) {
535  break;
536  case BPFISD::RET_FLAG:
537  return "BPFISD::RET_FLAG";
538  case BPFISD::CALL:
539  return "BPFISD::CALL";
540  case BPFISD::SELECT_CC:
541  return "BPFISD::SELECT_CC";
542  case BPFISD::BR_CC:
543  return "BPFISD::BR_CC";
544  case BPFISD::Wrapper:
545  return "BPFISD::Wrapper";
546  case BPFISD::MEMCPY:
547  return "BPFISD::MEMCPY";
548  }
549  return nullptr;
550 }
551 
552 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
553  SelectionDAG &DAG) const {
554  auto N = cast<GlobalAddressSDNode>(Op);
555  assert(N->getOffset() == 0 && "Invalid offset for global address");
556 
557  SDLoc DL(Op);
558  const GlobalValue *GV = N->getGlobal();
559  SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
560 
561  return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
562 }
563 
564 unsigned
565 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
566  unsigned Reg, bool isSigned) const {
569  int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
570  MachineFunction *F = BB->getParent();
571  DebugLoc DL = MI.getDebugLoc();
572 
573  MachineRegisterInfo &RegInfo = F->getRegInfo();
574  unsigned PromotedReg0 = RegInfo.createVirtualRegister(RC);
575  unsigned PromotedReg1 = RegInfo.createVirtualRegister(RC);
576  unsigned PromotedReg2 = RegInfo.createVirtualRegister(RC);
577  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
578  BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
579  .addReg(PromotedReg0).addImm(32);
580  BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
581  .addReg(PromotedReg1).addImm(32);
582 
583  return PromotedReg2;
584 }
585 
587 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
588  MachineBasicBlock *BB)
589  const {
590  MachineFunction *MF = MI.getParent()->getParent();
592  MachineInstrBuilder MIB(*MF, MI);
593  unsigned ScratchReg;
594 
595  // This function does custom insertion during lowering BPFISD::MEMCPY which
596  // only has two register operands from memcpy semantics, the copy source
597  // address and the copy destination address.
598  //
599  // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
600  // a third scratch register to serve as the destination register of load and
601  // source register of store.
602  //
603  // The scratch register here is with the Define | Dead | EarlyClobber flags.
604  // The EarlyClobber flag has the semantic property that the operand it is
605  // attached to is clobbered before the rest of the inputs are read. Hence it
606  // must be unique among the operands to the instruction. The Define flag is
607  // needed to coerce the machine verifier that an Undef value isn't a problem
608  // as we anyway is loading memory into it. The Dead flag is needed as the
609  // value in scratch isn't supposed to be used by any other instruction.
610  ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
611  MIB.addReg(ScratchReg,
613 
614  return BB;
615 }
616 
619  MachineBasicBlock *BB) const {
621  DebugLoc DL = MI.getDebugLoc();
622  unsigned Opc = MI.getOpcode();
623  bool isSelectRROp = (Opc == BPF::Select ||
624  Opc == BPF::Select_64_32 ||
625  Opc == BPF::Select_32 ||
626  Opc == BPF::Select_32_64);
627 
628  bool isMemcpyOp = Opc == BPF::MEMCPY;
629 
630 #ifndef NDEBUG
631  bool isSelectRIOp = (Opc == BPF::Select_Ri ||
632  Opc == BPF::Select_Ri_64_32 ||
633  Opc == BPF::Select_Ri_32 ||
634  Opc == BPF::Select_Ri_32_64);
635 
636 
637  assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
638  "Unexpected instr type to insert");
639 #endif
640 
641  if (isMemcpyOp)
642  return EmitInstrWithCustomInserterMemcpy(MI, BB);
643 
644  bool is32BitCmp = (Opc == BPF::Select_32 ||
645  Opc == BPF::Select_32_64 ||
646  Opc == BPF::Select_Ri_32 ||
647  Opc == BPF::Select_Ri_32_64);
648 
649  // To "insert" a SELECT instruction, we actually have to insert the diamond
650  // control-flow pattern. The incoming instruction knows the destination vreg
651  // to set, the condition code register to branch on, the true/false values to
652  // select between, and a branch opcode to use.
653  const BasicBlock *LLVM_BB = BB->getBasicBlock();
655 
656  // ThisMBB:
657  // ...
658  // TrueVal = ...
659  // jmp_XX r1, r2 goto Copy1MBB
660  // fallthrough --> Copy0MBB
661  MachineBasicBlock *ThisMBB = BB;
662  MachineFunction *F = BB->getParent();
663  MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
664  MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
665 
666  F->insert(I, Copy0MBB);
667  F->insert(I, Copy1MBB);
668  // Update machine-CFG edges by transferring all successors of the current
669  // block to the new block which will contain the Phi node for the select.
670  Copy1MBB->splice(Copy1MBB->begin(), BB,
671  std::next(MachineBasicBlock::iterator(MI)), BB->end());
672  Copy1MBB->transferSuccessorsAndUpdatePHIs(BB);
673  // Next, add the true and fallthrough blocks as its successors.
674  BB->addSuccessor(Copy0MBB);
675  BB->addSuccessor(Copy1MBB);
676 
677  // Insert Branch if Flag
678  int CC = MI.getOperand(3).getImm();
679  int NewCC;
680  switch (CC) {
681 #define SET_NEWCC(X, Y) \
682  case ISD::X: \
683  if (is32BitCmp && HasJmp32) \
684  NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
685  else \
686  NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
687  break
688  SET_NEWCC(SETGT, JSGT);
689  SET_NEWCC(SETUGT, JUGT);
690  SET_NEWCC(SETGE, JSGE);
691  SET_NEWCC(SETUGE, JUGE);
692  SET_NEWCC(SETEQ, JEQ);
693  SET_NEWCC(SETNE, JNE);
694  SET_NEWCC(SETLT, JSLT);
695  SET_NEWCC(SETULT, JULT);
696  SET_NEWCC(SETLE, JSLE);
697  SET_NEWCC(SETULE, JULE);
698  default:
699  report_fatal_error("unimplemented select CondCode " + Twine(CC));
700  }
701 
702  unsigned LHS = MI.getOperand(1).getReg();
703  bool isSignedCmp = (CC == ISD::SETGT ||
704  CC == ISD::SETGE ||
705  CC == ISD::SETLT ||
706  CC == ISD::SETLE);
707 
708  // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
709  // to be promoted, however if the 32-bit comparison operands are destination
710  // registers then they are implicitly zero-extended already, there is no
711  // need of explicit zero-extend sequence for them.
712  //
713  // We simply do extension for all situations in this method, but we will
714  // try to remove those unnecessary in BPFMIPeephole pass.
715  if (is32BitCmp && !HasJmp32)
716  LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
717 
718  if (isSelectRROp) {
719  unsigned RHS = MI.getOperand(2).getReg();
720 
721  if (is32BitCmp && !HasJmp32)
722  RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
723 
724  BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
725  } else {
726  int64_t imm32 = MI.getOperand(2).getImm();
727  // sanity check before we build J*_ri instruction.
728  assert (isInt<32>(imm32));
729  BuildMI(BB, DL, TII.get(NewCC))
730  .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
731  }
732 
733  // Copy0MBB:
734  // %FalseValue = ...
735  // # fallthrough to Copy1MBB
736  BB = Copy0MBB;
737 
738  // Update machine-CFG edges
739  BB->addSuccessor(Copy1MBB);
740 
741  // Copy1MBB:
742  // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
743  // ...
744  BB = Copy1MBB;
745  BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
746  .addReg(MI.getOperand(5).getReg())
747  .addMBB(Copy0MBB)
748  .addReg(MI.getOperand(4).getReg())
749  .addMBB(ThisMBB);
750 
751  MI.eraseFromParent(); // The pseudo instruction is gone now.
752  return BB;
753 }
754 
756  EVT VT) const {
757  return getHasAlu32() ? MVT::i32 : MVT::i64;
758 }
759 
761  EVT VT) const {
762  return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
763 }
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
EVT getValueType() const
Return the ValueType of the referenced return value.
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
Diagnostic information for unsupported feature in backend.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it&#39;s not CSE&#39;d)...
Definition: SelectionDAG.h:852
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:678
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void addLiveIn(unsigned Reg, unsigned vreg=0)
addLiveIn - Add the specified register as a live-in.
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:384
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getReg() const
getReg - Returns the register number.
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:730
unsigned Reg
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
#define SET_NEWCC(X, Y)
F(f)
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned char TargetFlags=0)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:459
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:592
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:480
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
SimpleValueType SimpleTy
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:410
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LocInfo getLocInfo() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SmallVector< ISD::InputArg, 32 > Ins
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:726
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:400
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
SmallVector< ISD::OutputArg, 32 > Outs
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:995
virtual const TargetInstrInfo * getInstrInfo() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:117
TargetInstrInfo - Interface to description of machine instruction set.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
unsigned const MachineRegisterInfo * MRI
Machine Value Type.
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:840
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned getCommonMaxStoresPerMemFunc() const
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition: ValueTypes.cpp:114
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:56
void setPrefFunctionAlignment(unsigned Align)
Set the target&#39;s preferred function alignment.
self_iterator getIterator()
Definition: ilist_node.h:81
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
unsigned MaxStoresPerMemmove
Specify maximum bytes of store instructions per memmove call.
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:440
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Extended Value Type.
Definition: ValueTypes.h:33
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This structure contains all information that is necessary for lowering calls.
size_t size() const
Definition: SmallVector.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:86
Iterator for intrusive lists based on ilist_node.
CCState - This class holds information needed while lowering arguments and return values...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:308
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
const DebugLoc & getDebugLoc() const
CCValAssign - Represent assignment of one arg/retval to a location.
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:672
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:257
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:437
bool getHasJmp32() const
Definition: BPFSubtarget.h:72
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
int64_t getImm() const
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:83
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:685
const Function & getFunction() const
Return the LLVM function that this machine code represents.
static mvt_range integer_valuetypes()
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
amdgpu Simplify well known AMD library false FunctionCallee Callee
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:444
void setMinFunctionAlignment(unsigned Align)
Set the target&#39;s minimum function alignment (in log2(bytes))
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:492
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:495
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:255
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:666
Representation of each machine instruction.
Definition: MachineInstr.h:63
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
SmallVector< SDValue, 32 > OutVals
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:711
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:510
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
unsigned MaxStoresPerMemmoveOptSize
Maximum number of store instructions that may be substituted for a call to memmove, used for functions with OptSize attribute.
unsigned MaxStoresPerMemcpyOptSize
Maximum number of store operations that may be substituted for a call to memcpy, used for functions w...
void setStackPointerRegisterToSaveRestore(unsigned R)
If set to a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save and restore.
bool getHasAlu32() const
Definition: BPFSubtarget.h:73
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
unsigned MaxStoresPerMemcpy
Specify maximum bytes of store instructions per memcpy call.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
bool isRegLoc() const
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:482
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the &#39;usesCustomInserter&#39; fla...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getValueType(EVT)
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:467
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
unsigned MaxStoresPerMemsetOptSize
Maximum number of stores operations that may be substituted for the call to memset, used for functions with OptSize attribute.
Conversion operators.
Definition: ISDOpcodes.h:489
const SDValue & getOperand(unsigned i) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getLocReg() const
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:498
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:415
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool getHasJmpExt() const
Definition: BPFSubtarget.h:71
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
LLVMContext * getContext() const
Definition: SelectionDAG.h:410
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:628
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:404
BRIND - Indirect branch.
Definition: ISDOpcodes.h:662
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:651