LLVM 17.0.0git
HexagonISelLowering.cpp
Go to the documentation of this file.
1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/CallingConv.h"
36#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InlineAsm.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/Type.h"
50#include "llvm/IR/Value.h"
55#include "llvm/Support/Debug.h"
60#include <algorithm>
61#include <cassert>
62#include <cstddef>
63#include <cstdint>
64#include <limits>
65#include <utility>
66
67using namespace llvm;
68
69#define DEBUG_TYPE "hexagon-lowering"
70
71static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
72 cl::init(true), cl::Hidden,
73 cl::desc("Control jump table emission on Hexagon target"));
74
75static cl::opt<bool>
76 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
77 cl::desc("Enable Hexagon SDNode scheduling"));
78
80 cl::desc("Enable Fast Math processing"));
81
82static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
83 cl::init(5),
84 cl::desc("Set minimum jump tables"));
85
86static cl::opt<int>
87 MaxStoresPerMemcpyCL("max-store-memcpy", cl::Hidden, cl::init(6),
88 cl::desc("Max #stores to inline memcpy"));
89
90static cl::opt<int>
92 cl::desc("Max #stores to inline memcpy"));
93
94static cl::opt<int>
95 MaxStoresPerMemmoveCL("max-store-memmove", cl::Hidden, cl::init(6),
96 cl::desc("Max #stores to inline memmove"));
97
98static cl::opt<int>
100 cl::init(4),
101 cl::desc("Max #stores to inline memmove"));
102
103static cl::opt<int>
104 MaxStoresPerMemsetCL("max-store-memset", cl::Hidden, cl::init(8),
105 cl::desc("Max #stores to inline memset"));
106
107static cl::opt<int>
109 cl::desc("Max #stores to inline memset"));
110
111static cl::opt<bool> AlignLoads("hexagon-align-loads",
112 cl::Hidden, cl::init(false),
113 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
114
115static cl::opt<bool>
116 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
117 cl::init(false),
118 cl::desc("Disable minimum alignment of 1 for "
119 "arguments passed by value on stack"));
120
121namespace {
122
123 class HexagonCCState : public CCState {
124 unsigned NumNamedVarArgParams = 0;
125
126 public:
127 HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
129 unsigned NumNamedArgs)
130 : CCState(CC, IsVarArg, MF, locs, C),
131 NumNamedVarArgParams(NumNamedArgs) {}
132 unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
133 };
134
135} // end anonymous namespace
136
137
138// Implement calling convention for Hexagon.
139
140static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
141 CCValAssign::LocInfo &LocInfo,
142 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
143 static const MCPhysReg ArgRegs[] = {
144 Hexagon::R0, Hexagon::R1, Hexagon::R2,
145 Hexagon::R3, Hexagon::R4, Hexagon::R5
146 };
147 const unsigned NumArgRegs = std::size(ArgRegs);
148 unsigned RegNum = State.getFirstUnallocated(ArgRegs);
149
150 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
151 if (RegNum != NumArgRegs && RegNum % 2 == 1)
152 State.AllocateReg(ArgRegs[RegNum]);
153
154 // Always return false here, as this function only makes sure that the first
155 // unallocated register has an even register number and does not actually
156 // allocate a register for the current argument.
157 return false;
158}
159
160#include "HexagonGenCallingConv.inc"
161
162
165 const {
166 return SDValue();
167}
168
169/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
170/// by "Src" to address "Dst" of size "Size". Alignment information is
171/// specified by the specific parameter attribute. The copy will be passed as
172/// a byval function parameter. Sometimes what we are copying is the end of a
173/// larger object, the part that does not fit in registers.
175 SDValue Chain, ISD::ArgFlagsTy Flags,
176 SelectionDAG &DAG, const SDLoc &dl) {
177 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
178 return DAG.getMemcpy(
179 Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
180 /*isVolatile=*/false, /*AlwaysInline=*/false,
181 /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
182}
183
184bool
186 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
188 LLVMContext &Context) const {
190 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
191
193 return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX);
194 return CCInfo.CheckReturn(Outs, RetCC_Hexagon);
195}
196
197// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
198// passed by value, the function prototype is modified to return void and
199// the value is stored in memory pointed by a pointer passed by caller.
202 bool IsVarArg,
204 const SmallVectorImpl<SDValue> &OutVals,
205 const SDLoc &dl, SelectionDAG &DAG) const {
206 // CCValAssign - represent the assignment of the return value to locations.
208
209 // CCState - Info about the registers and stack slot.
210 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
211 *DAG.getContext());
212
213 // Analyze return values of ISD::RET
214 if (Subtarget.useHVXOps())
215 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX);
216 else
217 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon);
218
219 SDValue Flag;
220 SmallVector<SDValue, 4> RetOps(1, Chain);
221
222 // Copy the result values into the output registers.
223 for (unsigned i = 0; i != RVLocs.size(); ++i) {
224 CCValAssign &VA = RVLocs[i];
225 SDValue Val = OutVals[i];
226
227 switch (VA.getLocInfo()) {
228 default:
229 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
230 llvm_unreachable("Unknown loc info!");
232 break;
234 Val = DAG.getBitcast(VA.getLocVT(), Val);
235 break;
237 Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val);
238 break;
240 Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val);
241 break;
243 Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val);
244 break;
245 }
246
247 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Flag);
248
249 // Guarantee that all emitted copies are stuck together with flags.
250 Flag = Chain.getValue(1);
251 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
252 }
253
254 RetOps[0] = Chain; // Update chain.
255
256 // Add the flag if we have it.
257 if (Flag.getNode())
258 RetOps.push_back(Flag);
259
260 return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps);
261}
262
264 // If either no tail call or told not to tail call at all, don't.
265 return CI->isTailCall();
266}
267
269 const char* RegName, LLT VT, const MachineFunction &) const {
270 // Just support r19, the linux kernel uses it.
272 .Case("r0", Hexagon::R0)
273 .Case("r1", Hexagon::R1)
274 .Case("r2", Hexagon::R2)
275 .Case("r3", Hexagon::R3)
276 .Case("r4", Hexagon::R4)
277 .Case("r5", Hexagon::R5)
278 .Case("r6", Hexagon::R6)
279 .Case("r7", Hexagon::R7)
280 .Case("r8", Hexagon::R8)
281 .Case("r9", Hexagon::R9)
282 .Case("r10", Hexagon::R10)
283 .Case("r11", Hexagon::R11)
284 .Case("r12", Hexagon::R12)
285 .Case("r13", Hexagon::R13)
286 .Case("r14", Hexagon::R14)
287 .Case("r15", Hexagon::R15)
288 .Case("r16", Hexagon::R16)
289 .Case("r17", Hexagon::R17)
290 .Case("r18", Hexagon::R18)
291 .Case("r19", Hexagon::R19)
292 .Case("r20", Hexagon::R20)
293 .Case("r21", Hexagon::R21)
294 .Case("r22", Hexagon::R22)
295 .Case("r23", Hexagon::R23)
296 .Case("r24", Hexagon::R24)
297 .Case("r25", Hexagon::R25)
298 .Case("r26", Hexagon::R26)
299 .Case("r27", Hexagon::R27)
300 .Case("r28", Hexagon::R28)
301 .Case("r29", Hexagon::R29)
302 .Case("r30", Hexagon::R30)
303 .Case("r31", Hexagon::R31)
304 .Case("r1:0", Hexagon::D0)
305 .Case("r3:2", Hexagon::D1)
306 .Case("r5:4", Hexagon::D2)
307 .Case("r7:6", Hexagon::D3)
308 .Case("r9:8", Hexagon::D4)
309 .Case("r11:10", Hexagon::D5)
310 .Case("r13:12", Hexagon::D6)
311 .Case("r15:14", Hexagon::D7)
312 .Case("r17:16", Hexagon::D8)
313 .Case("r19:18", Hexagon::D9)
314 .Case("r21:20", Hexagon::D10)
315 .Case("r23:22", Hexagon::D11)
316 .Case("r25:24", Hexagon::D12)
317 .Case("r27:26", Hexagon::D13)
318 .Case("r29:28", Hexagon::D14)
319 .Case("r31:30", Hexagon::D15)
320 .Case("sp", Hexagon::R29)
321 .Case("fp", Hexagon::R30)
322 .Case("lr", Hexagon::R31)
323 .Case("p0", Hexagon::P0)
324 .Case("p1", Hexagon::P1)
325 .Case("p2", Hexagon::P2)
326 .Case("p3", Hexagon::P3)
327 .Case("sa0", Hexagon::SA0)
328 .Case("lc0", Hexagon::LC0)
329 .Case("sa1", Hexagon::SA1)
330 .Case("lc1", Hexagon::LC1)
331 .Case("m0", Hexagon::M0)
332 .Case("m1", Hexagon::M1)
333 .Case("usr", Hexagon::USR)
334 .Case("ugp", Hexagon::UGP)
335 .Case("cs0", Hexagon::CS0)
336 .Case("cs1", Hexagon::CS1)
337 .Default(Register());
338 if (Reg)
339 return Reg;
340
341 report_fatal_error("Invalid register name global variable");
342}
343
344/// LowerCallResult - Lower the result values of an ISD::CALL into the
345/// appropriate copies out of appropriate physical registers. This assumes that
346/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
347/// being lowered. Returns a SDNode with the same number of values as the
348/// ISD::CALL.
350 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
351 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
353 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
354 // Assign locations to each value returned by this call.
356
357 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
358 *DAG.getContext());
359
360 if (Subtarget.useHVXOps())
361 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX);
362 else
363 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon);
364
365 // Copy all of the result registers out of their specified physreg.
366 for (unsigned i = 0; i != RVLocs.size(); ++i) {
367 SDValue RetVal;
368 if (RVLocs[i].getValVT() == MVT::i1) {
369 // Return values of type MVT::i1 require special handling. The reason
370 // is that MVT::i1 is associated with the PredRegs register class, but
371 // values of that type are still returned in R0. Generate an explicit
372 // copy into a predicate register from R0, and treat the value of the
373 // predicate register as the call result.
374 auto &MRI = DAG.getMachineFunction().getRegInfo();
375 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
376 MVT::i32, Glue);
377 // FR0 = (Value, Chain, Glue)
378 Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);
379 SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,
380 FR0.getValue(0), FR0.getValue(2));
381 // TPR = (Chain, Glue)
382 // Don't glue this CopyFromReg, because it copies from a virtual
383 // register. If it is glued to the call, InstrEmitter will add it
384 // as an implicit def to the call (EmitMachineNode).
385 RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1);
386 Glue = TPR.getValue(1);
387 Chain = TPR.getValue(0);
388 } else {
389 RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
390 RVLocs[i].getValVT(), Glue);
391 Glue = RetVal.getValue(2);
392 Chain = RetVal.getValue(1);
393 }
394 InVals.push_back(RetVal.getValue(0));
395 }
396
397 return Chain;
398}
399
400/// LowerCall - Functions arguments are copied from virtual regs to
401/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
404 SmallVectorImpl<SDValue> &InVals) const {
405 SelectionDAG &DAG = CLI.DAG;
406 SDLoc &dl = CLI.DL;
408 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
410 SDValue Chain = CLI.Chain;
411 SDValue Callee = CLI.Callee;
412 CallingConv::ID CallConv = CLI.CallConv;
413 bool IsVarArg = CLI.IsVarArg;
414 bool DoesNotReturn = CLI.DoesNotReturn;
415
416 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
418 MachineFrameInfo &MFI = MF.getFrameInfo();
419 auto PtrVT = getPointerTy(MF.getDataLayout());
420
421 unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0;
422 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))
423 Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);
424
425 // Linux ABI treats var-arg calls the same way as regular ones.
426 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
427
428 // Analyze operands of the call, assigning locations to each operand.
430 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(),
431 NumParams);
432
433 if (Subtarget.useHVXOps())
434 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
436 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
437 else
438 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
439
440 if (CLI.IsTailCall) {
441 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
443 IsVarArg, IsStructRet, StructAttrFlag, Outs,
444 OutVals, Ins, DAG);
445 for (const CCValAssign &VA : ArgLocs) {
446 if (VA.isMemLoc()) {
447 CLI.IsTailCall = false;
448 break;
449 }
450 }
451 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
452 : "Argument must be passed on stack. "
453 "Not eligible for Tail Call\n"));
454 }
455 // Get a count of how many bytes are to be pushed on the stack.
456 unsigned NumBytes = CCInfo.getNextStackOffset();
458 SmallVector<SDValue, 8> MemOpChains;
459
460 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
461 SDValue StackPtr =
462 DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);
463
464 bool NeedsArgAlign = false;
465 Align LargestAlignSeen;
466 // Walk the register/memloc assignments, inserting copies/loads.
467 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
468 CCValAssign &VA = ArgLocs[i];
469 SDValue Arg = OutVals[i];
470 ISD::ArgFlagsTy Flags = Outs[i].Flags;
471 // Record if we need > 8 byte alignment on an argument.
472 bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT());
473 NeedsArgAlign |= ArgAlign;
474
475 // Promote the value if needed.
476 switch (VA.getLocInfo()) {
477 default:
478 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
479 llvm_unreachable("Unknown loc info!");
481 break;
483 Arg = DAG.getBitcast(VA.getLocVT(), Arg);
484 break;
486 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
487 break;
489 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
490 break;
492 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
493 break;
494 }
495
496 if (VA.isMemLoc()) {
497 unsigned LocMemOffset = VA.getLocMemOffset();
498 SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
499 StackPtr.getValueType());
500 MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);
501 if (ArgAlign)
502 LargestAlignSeen = std::max(
503 LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8));
504 if (Flags.isByVal()) {
505 // The argument is a struct passed by value. According to LLVM, "Arg"
506 // is a pointer.
507 MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,
508 Flags, DAG, dl));
509 } else {
511 DAG.getMachineFunction(), LocMemOffset);
512 SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);
513 MemOpChains.push_back(S);
514 }
515 continue;
516 }
517
518 // Arguments that can be passed on register must be kept at RegsToPass
519 // vector.
520 if (VA.isRegLoc())
521 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
522 }
523
524 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
525 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
526 Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
527 LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);
528 MFI.ensureMaxAlignment(LargestAlignSeen);
529 }
530 // Transform all store nodes into one single node because all store
531 // nodes are independent of each other.
532 if (!MemOpChains.empty())
533 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
534
535 SDValue Glue;
536 if (!CLI.IsTailCall) {
537 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
538 Glue = Chain.getValue(1);
539 }
540
541 // Build a sequence of copy-to-reg nodes chained together with token
542 // chain and flag operands which copy the outgoing args into registers.
543 // The Glue is necessary since all emitted instructions must be
544 // stuck together.
545 if (!CLI.IsTailCall) {
546 for (const auto &R : RegsToPass) {
547 Chain = DAG.getCopyToReg(Chain, dl, R.first, R.second, Glue);
548 Glue = Chain.getValue(1);
549 }
550 } else {
551 // For tail calls lower the arguments to the 'real' stack slot.
552 //
553 // Force all the incoming stack arguments to be loaded from the stack
554 // before any new outgoing arguments are stored to the stack, because the
555 // outgoing stack slots may alias the incoming argument stack slots, and
556 // the alias isn't otherwise explicit. This is slightly more conservative
557 // than necessary, because it means that each store effectively depends
558 // on every argument instead of just those arguments it would clobber.
559 //
560 // Do not flag preceding copytoreg stuff together with the following stuff.
561 Glue = SDValue();
562 for (const auto &R : RegsToPass) {
563 Chain = DAG.getCopyToReg(Chain, dl, R.first, R.second, Glue);
564 Glue = Chain.getValue(1);
565 }
566 Glue = SDValue();
567 }
568
569 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
570 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
571
572 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
573 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
574 // node so that legalize doesn't hack it.
575 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
576 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags);
577 } else if (ExternalSymbolSDNode *S =
578 dyn_cast<ExternalSymbolSDNode>(Callee)) {
579 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags);
580 }
581
582 // Returns a chain & a flag for retval copy to use.
583 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
585 Ops.push_back(Chain);
586 Ops.push_back(Callee);
587
588 // Add argument registers to the end of the list so that they are
589 // known live into the call.
590 for (const auto &R : RegsToPass)
591 Ops.push_back(DAG.getRegister(R.first, R.second.getValueType()));
592
593 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
594 assert(Mask && "Missing call preserved mask for calling convention");
595 Ops.push_back(DAG.getRegisterMask(Mask));
596
597 if (Glue.getNode())
598 Ops.push_back(Glue);
599
600 if (CLI.IsTailCall) {
601 MFI.setHasTailCall();
602 return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops);
603 }
604
605 // Set this here because we need to know this for "hasFP" in frame lowering.
606 // The target-independent code calls getFrameRegister before setting it, and
607 // getFrameRegister uses hasFP to determine whether the function has FP.
608 MFI.setHasCalls(true);
609
610 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
611 Chain = DAG.getNode(OpCode, dl, NodeTys, Ops);
612 Glue = Chain.getValue(1);
613
614 // Create the CALLSEQ_END node.
615 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, dl);
616 Glue = Chain.getValue(1);
617
618 // Handle result values, copying them out of physregs into vregs that we
619 // return.
620 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
621 InVals, OutVals, Callee);
622}
623
624/// Returns true by value, base pointer and offset pointer and addressing
625/// mode by reference if this node can be combined with a load / store to
626/// form a post-indexed load / store.
629 SelectionDAG &DAG) const {
630 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N);
631 if (!LSN)
632 return false;
633 EVT VT = LSN->getMemoryVT();
634 if (!VT.isSimple())
635 return false;
636 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
637 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
638 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
639 VT == MVT::v4i16 || VT == MVT::v8i8 ||
640 Subtarget.isHVXVectorType(VT.getSimpleVT());
641 if (!IsLegalType)
642 return false;
643
644 if (Op->getOpcode() != ISD::ADD)
645 return false;
646 Base = Op->getOperand(0);
647 Offset = Op->getOperand(1);
648 if (!isa<ConstantSDNode>(Offset.getNode()))
649 return false;
650 AM = ISD::POST_INC;
651
652 int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
653 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V);
654}
655
659 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
660 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
661 unsigned LR = HRI.getRARegister();
662
663 if ((Op.getOpcode() != ISD::INLINEASM &&
664 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
665 return Op;
666
667 unsigned NumOps = Op.getNumOperands();
668 if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue)
669 --NumOps; // Ignore the flag operand.
670
671 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
672 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
673 unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
674 ++i; // Skip the ID value.
675
676 switch (InlineAsm::getKind(Flags)) {
677 default:
678 llvm_unreachable("Bad flags!");
682 i += NumVals;
683 break;
687 for (; NumVals; --NumVals, ++i) {
688 Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
689 if (Reg != LR)
690 continue;
691 HMFI.setHasClobberLR(true);
692 return Op;
693 }
694 break;
695 }
696 }
697 }
698
699 return Op;
700}
701
702// Need to transform ISD::PREFETCH into something that doesn't inherit
703// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
704// SDNPMayStore.
706 SelectionDAG &DAG) const {
707 SDValue Chain = Op.getOperand(0);
708 SDValue Addr = Op.getOperand(1);
709 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
710 // if the "reg" is fed by an "add".
711 SDLoc DL(Op);
712 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
713 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
714}
715
716// Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode
717// is marked as having side-effects, while the register read on Hexagon does
718// not have any. TableGen refuses to accept the direct pattern from that node
719// to the A4_tfrcpp.
721 SelectionDAG &DAG) const {
722 SDValue Chain = Op.getOperand(0);
723 SDLoc dl(Op);
725 return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);
726}
727
729 SelectionDAG &DAG) const {
730 SDValue Chain = Op.getOperand(0);
731 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
732 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
733 if (IntNo == Intrinsic::hexagon_prefetch) {
734 SDValue Addr = Op.getOperand(2);
735 SDLoc DL(Op);
736 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
737 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
738 }
739 return SDValue();
740}
741
744 SelectionDAG &DAG) const {
745 SDValue Chain = Op.getOperand(0);
746 SDValue Size = Op.getOperand(1);
747 SDValue Align = Op.getOperand(2);
748 SDLoc dl(Op);
749
750 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);
751 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
752
753 unsigned A = AlignConst->getSExtValue();
754 auto &HFI = *Subtarget.getFrameLowering();
755 // "Zero" means natural stack alignment.
756 if (A == 0)
757 A = HFI.getStackAlign().value();
758
759 LLVM_DEBUG({
760 dbgs () << __func__ << " Align: " << A << " Size: ";
761 Size.getNode()->dump(&DAG);
762 dbgs() << "\n";
763 });
764
765 SDValue AC = DAG.getConstant(A, dl, MVT::i32);
767 SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
768
769 DAG.ReplaceAllUsesOfValueWith(Op, AA);
770 return AA;
771}
772
774 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
775 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
776 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
778 MachineFrameInfo &MFI = MF.getFrameInfo();
780
781 // Linux ABI treats var-arg calls the same way as regular ones.
782 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
783
784 // Assign locations to all of the incoming arguments.
786 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
787 *DAG.getContext(),
789
790 if (Subtarget.useHVXOps())
791 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
793 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
794 else
795 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
796
797 // For LLVM, in the case when returning a struct by value (>8byte),
798 // the first argument is a pointer that points to the location on caller's
799 // stack where the return value will be stored. For Hexagon, the location on
800 // caller's stack is passed only when the struct size is smaller than (and
801 // equal to) 8 bytes. If not, no address will be passed into callee and
802 // callee return the result direclty through R0/R1.
803 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
804 switch (RC.getID()) {
805 case Hexagon::IntRegsRegClassID:
806 return Reg - Hexagon::R0 + 1;
807 case Hexagon::DoubleRegsRegClassID:
808 return (Reg - Hexagon::D0 + 1) * 2;
809 case Hexagon::HvxVRRegClassID:
810 return Reg - Hexagon::V0 + 1;
811 case Hexagon::HvxWRRegClassID:
812 return (Reg - Hexagon::W0 + 1) * 2;
813 }
814 llvm_unreachable("Unexpected register class");
815 };
816
817 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
818 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
819 HFL.FirstVarArgSavedReg = 0;
821
822 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
823 CCValAssign &VA = ArgLocs[i];
824 ISD::ArgFlagsTy Flags = Ins[i].Flags;
825 bool ByVal = Flags.isByVal();
826
827 // Arguments passed in registers:
828 // 1. 32- and 64-bit values and HVX vectors are passed directly,
829 // 2. Large structs are passed via an address, and the address is
830 // passed in a register.
831 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
832 llvm_unreachable("ByValSize must be bigger than 8 bytes");
833
834 bool InReg = VA.isRegLoc() &&
835 (!ByVal || (ByVal && Flags.getByValSize() > 8));
836
837 if (InReg) {
838 MVT RegVT = VA.getLocVT();
839 if (VA.getLocInfo() == CCValAssign::BCvt)
840 RegVT = VA.getValVT();
841
842 const TargetRegisterClass *RC = getRegClassFor(RegVT);
843 Register VReg = MRI.createVirtualRegister(RC);
844 SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
845
846 // Treat values of type MVT::i1 specially: they are passed in
847 // registers of type i32, but they need to remain as values of
848 // type i1 for consistency of the argument lowering.
849 if (VA.getValVT() == MVT::i1) {
850 assert(RegVT.getSizeInBits() <= 32);
851 SDValue T = DAG.getNode(ISD::AND, dl, RegVT,
852 Copy, DAG.getConstant(1, dl, RegVT));
853 Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT),
854 ISD::SETNE);
855 } else {
856#ifndef NDEBUG
857 unsigned RegSize = RegVT.getSizeInBits();
858 assert(RegSize == 32 || RegSize == 64 ||
859 Subtarget.isHVXVectorType(RegVT));
860#endif
861 }
862 InVals.push_back(Copy);
863 MRI.addLiveIn(VA.getLocReg(), VReg);
864 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
865 } else {
866 assert(VA.isMemLoc() && "Argument should be passed in memory");
867
868 // If it's a byval parameter, then we need to compute the
869 // "real" size, not the size of the pointer.
870 unsigned ObjSize = Flags.isByVal()
871 ? Flags.getByValSize()
872 : VA.getLocVT().getStoreSizeInBits() / 8;
873
874 // Create the frame index object for this incoming parameter.
876 int FI = MFI.CreateFixedObject(ObjSize, Offset, true);
877 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
878
879 if (Flags.isByVal()) {
880 // If it's a pass-by-value aggregate, then do not dereference the stack
881 // location. Instead, we should generate a reference to the stack
882 // location.
883 InVals.push_back(FIN);
884 } else {
885 SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
887 InVals.push_back(L);
888 }
889 }
890 }
891
892 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
893 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
894 MRI.addLiveIn(Hexagon::R0+i);
895 }
896
897 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
898 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
899 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
900
901 // Create Frame index for the start of register saved area.
902 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
903 bool RequiresPadding = (NumVarArgRegs & 1);
904 int RegSaveAreaSizePlusPadding = RequiresPadding
905 ? (NumVarArgRegs + 1) * 4
906 : NumVarArgRegs * 4;
907
908 if (RegSaveAreaSizePlusPadding > 0) {
909 // The offset to saved register area should be 8 byte aligned.
910 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
911 if (!(RegAreaStart % 8))
912 RegAreaStart = (RegAreaStart + 7) & -8;
913
914 int RegSaveAreaFrameIndex =
915 MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true);
916 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
917
918 // This will point to the next argument passed via stack.
919 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
920 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
921 HMFI.setVarArgsFrameIndex(FI);
922 } else {
923 // This will point to the next argument passed via stack, when
924 // there is no saved register area.
925 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
926 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
927 HMFI.setRegSavedAreaStartFrameIndex(FI);
928 HMFI.setVarArgsFrameIndex(FI);
929 }
930 }
931
932
933 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
934 // This will point to the next argument passed via stack.
935 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
936 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
937 HMFI.setVarArgsFrameIndex(FI);
938 }
939
940 return Chain;
941}
942
945 // VASTART stores the address of the VarArgsFrameIndex slot into the
946 // memory location argument.
950 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
951
952 if (!Subtarget.isEnvironmentMusl()) {
953 return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1),
955 }
957 auto &HFL = *Subtarget.getFrameLowering();
958 SDLoc DL(Op);
960
961 // Get frame index of va_list.
962 SDValue FIN = Op.getOperand(1);
963
964 // If first Vararg register is odd, add 4 bytes to start of
965 // saved register area to point to the first register location.
966 // This is because the saved register area has to be 8 byte aligned.
967 // Incase of an odd start register, there will be 4 bytes of padding in
968 // the beginning of saved register area. If all registers area used up,
969 // the following condition will handle it correctly.
970 SDValue SavedRegAreaStartFrameIndex =
971 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32);
972
973 auto PtrVT = getPointerTy(DAG.getDataLayout());
974
975 if (HFL.FirstVarArgSavedReg & 1)
976 SavedRegAreaStartFrameIndex =
977 DAG.getNode(ISD::ADD, DL, PtrVT,
978 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(),
979 MVT::i32),
980 DAG.getIntPtrConstant(4, DL));
981
982 // Store the saved register area start pointer.
983 SDValue Store =
984 DAG.getStore(Op.getOperand(0), DL,
985 SavedRegAreaStartFrameIndex,
986 FIN, MachinePointerInfo(SV));
987 MemOps.push_back(Store);
988
989 // Store saved register area end pointer.
990 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
991 FIN, DAG.getIntPtrConstant(4, DL));
992 Store = DAG.getStore(Op.getOperand(0), DL,
993 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
994 PtrVT),
995 FIN, MachinePointerInfo(SV, 4));
996 MemOps.push_back(Store);
997
998 // Store overflow area pointer.
999 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
1000 FIN, DAG.getIntPtrConstant(4, DL));
1001 Store = DAG.getStore(Op.getOperand(0), DL,
1002 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
1003 PtrVT),
1004 FIN, MachinePointerInfo(SV, 8));
1005 MemOps.push_back(Store);
1006
1007 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
1008}
1009
1010SDValue
1012 // Assert that the linux ABI is enabled for the current compilation.
1013 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1014 SDValue Chain = Op.getOperand(0);
1015 SDValue DestPtr = Op.getOperand(1);
1016 SDValue SrcPtr = Op.getOperand(2);
1017 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
1018 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
1019 SDLoc DL(Op);
1020 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1021 // we need to memcopy 12 bytes from va_list to another similar list.
1022 return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
1023 DAG.getIntPtrConstant(12, DL), Align(4),
1024 /*isVolatile*/ false, false, false,
1025 MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
1026}
1027
1029 const SDLoc &dl(Op);
1030 SDValue LHS = Op.getOperand(0);
1031 SDValue RHS = Op.getOperand(1);
1032 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1033 MVT ResTy = ty(Op);
1034 MVT OpTy = ty(LHS);
1035
1036 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1037 MVT ElemTy = OpTy.getVectorElementType();
1038 assert(ElemTy.isScalarInteger());
1040 OpTy.getVectorNumElements());
1041 return DAG.getSetCC(dl, ResTy,
1042 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy),
1043 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC);
1044 }
1045
1046 // Treat all other vector types as legal.
1047 if (ResTy.isVector())
1048 return Op;
1049
1050 // Comparisons of short integers should use sign-extend, not zero-extend,
1051 // since we can represent small negative values in the compare instructions.
1052 // The LLVM default is to use zero-extend arbitrarily in these cases.
1053 auto isSExtFree = [this](SDValue N) {
1054 switch (N.getOpcode()) {
1055 case ISD::TRUNCATE: {
1056 // A sign-extend of a truncate of a sign-extend is free.
1057 SDValue Op = N.getOperand(0);
1058 if (Op.getOpcode() != ISD::AssertSext)
1059 return false;
1060 EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();
1061 unsigned ThisBW = ty(N).getSizeInBits();
1062 unsigned OrigBW = OrigTy.getSizeInBits();
1063 // The type that was sign-extended to get the AssertSext must be
1064 // narrower than the type of N (so that N has still the same value
1065 // as the original).
1066 return ThisBW >= OrigBW;
1067 }
1068 case ISD::LOAD:
1069 // We have sign-extended loads.
1070 return true;
1071 }
1072 return false;
1073 };
1074
1075 if (OpTy == MVT::i8 || OpTy == MVT::i16) {
1076 ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS);
1077 bool IsNegative = C && C->getAPIntValue().isNegative();
1078 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1079 return DAG.getSetCC(dl, ResTy,
1082 }
1083
1084 return SDValue();
1085}
1086
1087SDValue
1089 SDValue PredOp = Op.getOperand(0);
1090 SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
1091 MVT OpTy = ty(Op1);
1092 const SDLoc &dl(Op);
1093
1094 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1095 MVT ElemTy = OpTy.getVectorElementType();
1096 assert(ElemTy.isScalarInteger());
1098 OpTy.getVectorNumElements());
1099 // Generate (trunc (select (_, sext, sext))).
1100 return DAG.getSExtOrTrunc(
1101 DAG.getSelect(dl, WideTy, PredOp,
1102 DAG.getSExtOrTrunc(Op1, dl, WideTy),
1103 DAG.getSExtOrTrunc(Op2, dl, WideTy)),
1104 dl, OpTy);
1105 }
1106
1107 return SDValue();
1108}
1109
1110SDValue
1112 EVT ValTy = Op.getValueType();
1113 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op);
1114 Constant *CVal = nullptr;
1115 bool isVTi1Type = false;
1116 if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
1117 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1118 IRBuilder<> IRB(CV->getContext());
1120 unsigned VecLen = CV->getNumOperands();
1121 assert(isPowerOf2_32(VecLen) &&
1122 "conversion only supported for pow2 VectorSize");
1123 for (unsigned i = 0; i < VecLen; ++i)
1124 NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue()));
1125
1126 CVal = ConstantVector::get(NewConst);
1127 isVTi1Type = true;
1128 }
1129 }
1130 Align Alignment = CPN->getAlign();
1131 bool IsPositionIndependent = isPositionIndependent();
1132 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1133
1134 unsigned Offset = 0;
1135 SDValue T;
1136 if (CPN->isMachineConstantPoolEntry())
1137 T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment,
1138 Offset, TF);
1139 else if (isVTi1Type)
1140 T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF);
1141 else
1142 T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset,
1143 TF);
1144
1145 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1146 "Inconsistent target flag encountered");
1147
1148 if (IsPositionIndependent)
1149 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T);
1150 return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T);
1151}
1152
1153SDValue
1155 EVT VT = Op.getValueType();
1156 int Idx = cast<JumpTableSDNode>(Op)->getIndex();
1157 if (isPositionIndependent()) {
1159 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T);
1160 }
1161
1162 SDValue T = DAG.getTargetJumpTable(Idx, VT);
1163 return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T);
1164}
1165
1166SDValue
1168 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1170 MachineFrameInfo &MFI = MF.getFrameInfo();
1171 MFI.setReturnAddressIsTaken(true);
1172
1174 return SDValue();
1175
1176 EVT VT = Op.getValueType();
1177 SDLoc dl(Op);
1178 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1179 if (Depth) {
1180 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1181 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
1182 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
1183 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
1185 }
1186
1187 // Return LR, which contains the return address. Mark it an implicit live-in.
1188 Register Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));
1189 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
1190}
1191
1192SDValue
1194 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1196 MFI.setFrameAddressIsTaken(true);
1197
1198 EVT VT = Op.getValueType();
1199 SDLoc dl(Op);
1200 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1201 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1202 HRI.getFrameRegister(), VT);
1203 while (Depth--)
1204 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
1206 return FrameAddr;
1207}
1208
1209SDValue
1211 SDLoc dl(Op);
1212 return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0));
1213}
1214
1215SDValue
1217 SDLoc dl(Op);
1218 auto *GAN = cast<GlobalAddressSDNode>(Op);
1219 auto PtrVT = getPointerTy(DAG.getDataLayout());
1220 auto *GV = GAN->getGlobal();
1221 int64_t Offset = GAN->getOffset();
1222
1223 auto &HLOF = *HTM.getObjFileLowering();
1225
1226 if (RM == Reloc::Static) {
1227 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
1228 const GlobalObject *GO = GV->getAliaseeObject();
1229 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1230 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA);
1231 return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA);
1232 }
1233
1234 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
1235 if (UsePCRel) {
1236 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,
1238 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA);
1239 }
1240
1241 // Use GOT index.
1242 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
1243 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);
1244 SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);
1245 return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off);
1246}
1247
1248// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1249SDValue
1251 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
1252 SDLoc dl(Op);
1253 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1254
1256 if (RM == Reloc::Static) {
1257 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);
1258 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A);
1259 }
1260
1262 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A);
1263}
1264
1265SDValue
1267 const {
1268 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1271 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym);
1272}
1273
1274SDValue
1276 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1277 unsigned char OperandFlags) const {
1279 MachineFrameInfo &MFI = MF.getFrameInfo();
1280 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1281 SDLoc dl(GA);
1282 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
1283 GA->getValueType(0),
1284 GA->getOffset(),
1285 OperandFlags);
1286 // Create Operands for the call.The Operands should have the following:
1287 // 1. Chain SDValue
1288 // 2. Callee which in this case is the Global address value.
1289 // 3. Registers live into the call.In this case its R0, as we
1290 // have just one argument to be passed.
1291 // 4. Glue.
1292 // Note: The order is important.
1293
1294 const auto &HRI = *Subtarget.getRegisterInfo();
1295 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1296 assert(Mask && "Missing call preserved mask for calling convention");
1297 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),
1298 DAG.getRegisterMask(Mask), Glue };
1299 Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops);
1300
1301 // Inform MFI that function has calls.
1302 MFI.setAdjustsStack(true);
1303
1304 Glue = Chain.getValue(1);
1305 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
1306}
1307
1308//
1309// Lower using the intial executable model for TLS addresses
1310//
1311SDValue
1313 SelectionDAG &DAG) const {
1314 SDLoc dl(GA);
1315 int64_t Offset = GA->getOffset();
1316 auto PtrVT = getPointerTy(DAG.getDataLayout());
1317
1318 // Get the thread pointer.
1319 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1320
1321 bool IsPositionIndependent = isPositionIndependent();
1322 unsigned char TF =
1323 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1324
1325 // First generate the TLS symbol address
1326 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,
1327 Offset, TF);
1328
1329 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1330
1331 if (IsPositionIndependent) {
1332 // Generate the GOT pointer in case of position independent code
1333 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);
1334
1335 // Add the TLS Symbol address to GOT pointer.This gives
1336 // GOT relative relocation for the symbol.
1337 Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1338 }
1339
1340 // Load the offset value for TLS symbol.This offset is relative to
1341 // thread pointer.
1342 SDValue LoadOffset =
1343 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo());
1344
1345 // Address of the thread local variable is the add of thread
1346 // pointer and the offset of the variable.
1347 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset);
1348}
1349
1350//
1351// Lower using the local executable model for TLS addresses
1352//
1353SDValue
1355 SelectionDAG &DAG) const {
1356 SDLoc dl(GA);
1357 int64_t Offset = GA->getOffset();
1358 auto PtrVT = getPointerTy(DAG.getDataLayout());
1359
1360 // Get the thread pointer.
1361 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1362 // Generate the TLS symbol address
1363 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1365 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1366
1367 // Address of the thread local variable is the add of thread
1368 // pointer and the offset of the variable.
1369 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym);
1370}
1371
1372//
1373// Lower using the general dynamic model for TLS addresses
1374//
1375SDValue
1377 SelectionDAG &DAG) const {
1378 SDLoc dl(GA);
1379 int64_t Offset = GA->getOffset();
1380 auto PtrVT = getPointerTy(DAG.getDataLayout());
1381
1382 // First generate the TLS symbol address
1383 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1385
1386 // Then, generate the GOT pointer
1387 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);
1388
1389 // Add the TLS symbol and the GOT pointer
1390 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1391 SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1392
1393 // Copy over the argument to R0
1394 SDValue InFlag;
1395 Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag);
1396 InFlag = Chain.getValue(1);
1397
1398 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1401
1402 return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT,
1403 Hexagon::R0, Flags);
1404}
1405
1406//
1407// Lower TLS addresses.
1408//
1409// For now for dynamic models, we only support the general dynamic model.
1410//
1411SDValue
1413 SelectionDAG &DAG) const {
1414 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
1415
1416 switch (HTM.getTLSModel(GA->getGlobal())) {
1419 return LowerToTLSGeneralDynamicModel(GA, DAG);
1421 return LowerToTLSInitialExecModel(GA, DAG);
1423 return LowerToTLSLocalExecModel(GA, DAG);
1424 }
1425 llvm_unreachable("Bogus TLS model");
1426}
1427
1428//===----------------------------------------------------------------------===//
1429// TargetLowering Implementation
1430//===----------------------------------------------------------------------===//
1431
1433 const HexagonSubtarget &ST)
1434 : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)),
1435 Subtarget(ST) {
1436 auto &HRI = *Subtarget.getRegisterInfo();
1437
1441 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1444
1447
1450 else
1452
1453 // Limits for inline expansion of memcpy/memmove
1460
1461 //
1462 // Set up register classes.
1463 //
1464
1465 addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass);
1466 addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa
1467 addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa
1468 addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba
1469 addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass);
1470 addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);
1471 addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass);
1472 addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass);
1473 addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass);
1474 addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
1475 addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
1476
1477 addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);
1478 addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);
1479
1480 //
1481 // Handling of scalar operations.
1482 //
1483 // All operations default to "legal", except:
1484 // - indexed loads and stores (pre-/post-incremented),
1485 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1486 // ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1487 // FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
1488 // FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
1489 // which default to "expand" for at least one type.
1490
1491 // Misc operations.
1508
1509 // Custom legalize GlobalAddress nodes into CONST32.
1513
1514 // Hexagon needs to optimize cases with negative constants.
1519
1520 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1524 if (Subtarget.isEnvironmentMusl())
1526 else
1528
1532
1533 if (EmitJumpTables)
1535 else
1536 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1538
1539 for (unsigned LegalIntOp :
1541 setOperationAction(LegalIntOp, MVT::i32, Legal);
1542 setOperationAction(LegalIntOp, MVT::i64, Legal);
1543 }
1544
1545 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1546 // but they only operate on i64.
1547 for (MVT VT : MVT::integer_valuetypes()) {
1554 }
1557
1562
1563 // Popcount can count # of 1s in i64 but returns i32.
1568
1573
1578
1579 for (unsigned IntExpOp :
1584 for (MVT VT : MVT::integer_valuetypes())
1585 setOperationAction(IntExpOp, VT, Expand);
1586 }
1587
1588 for (unsigned FPExpOp :
1591 for (MVT VT : MVT::fp_valuetypes())
1592 setOperationAction(FPExpOp, VT, Expand);
1593 }
1594
1595 // No extending loads from i32.
1596 for (MVT VT : MVT::integer_valuetypes()) {
1600 }
1601 // Turn FP truncstore into trunc + store.
1603 // Turn FP extload into load/fpextend.
1604 for (MVT VT : MVT::fp_valuetypes())
1606
1607 // Expand BR_CC and SELECT_CC for all integer and fp types.
1608 for (MVT VT : MVT::integer_valuetypes()) {
1611 }
1612 for (MVT VT : MVT::fp_valuetypes()) {
1615 }
1617
1618 //
1619 // Handling of vector operations.
1620 //
1621
1622 // Set the action for vector operations to "expand", then override it with
1623 // either "custom" or "legal" for specific cases.
1624 static const unsigned VectExpOps[] = {
1625 // Integer arithmetic:
1629 // Logical/bit:
1632 // Floating point arithmetic/math functions:
1639 // Misc:
1641 // Vector:
1647 };
1648
1650 for (unsigned VectExpOp : VectExpOps)
1651 setOperationAction(VectExpOp, VT, Expand);
1652
1653 // Expand all extending loads and truncating stores:
1654 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1655 if (TargetVT == VT)
1656 continue;
1657 setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
1658 setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand);
1659 setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand);
1660 setTruncStoreAction(VT, TargetVT, Expand);
1661 }
1662
1663 // Normalize all inputs to SELECT to be vectors of i32.
1664 if (VT.getVectorElementType() != MVT::i32) {
1665 MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);
1667 AddPromotedToType(ISD::SELECT, VT, VT32);
1668 }
1672 }
1673
1674 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1675 // are legal.
1682
1686
1687 // Types natively supported:
1688 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1696
1697 setOperationAction(ISD::ADD, NativeVT, Legal);
1698 setOperationAction(ISD::SUB, NativeVT, Legal);
1699 setOperationAction(ISD::MUL, NativeVT, Legal);
1700 setOperationAction(ISD::AND, NativeVT, Legal);
1701 setOperationAction(ISD::OR, NativeVT, Legal);
1702 setOperationAction(ISD::XOR, NativeVT, Legal);
1703
1704 if (NativeVT.getVectorElementType() != MVT::i1)
1706 }
1707
1708 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1713 }
1714
1715 // Custom lower unaligned loads.
1716 // Also, for both loads and stores, verify the alignment of the address
1717 // in case it is a compile-time constant. This is a usability feature to
1718 // provide a meaningful error message to users.
1723 }
1724
1725 // Custom-lower load/stores of boolean vectors.
1726 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1729 }
1730
1732 MVT::v2i32}) {
1740 }
1741
1742 // Custom-lower bitcasts from i8 to v8i1.
1750
1751 // V5+.
1756
1759
1772
1773 // Special handling for half-precision floating point conversions.
1774 // Lower half float conversions into library calls.
1779
1784
1785 // Handling of indexed loads/stores: default is "expand".
1786 //
1791 }
1792
1793 // Subtarget-specific operation actions.
1794 //
1795 if (Subtarget.hasV60Ops()) {
1800 }
1801 if (Subtarget.hasV66Ops()) {
1804 }
1805 if (Subtarget.hasV67Ops()) {
1809 }
1810
1814
1815 if (Subtarget.useHVXOps())
1816 initializeHVXLowering();
1817
1819
1820 //
1821 // Library calls for unsupported operations
1822 //
1823 bool FastMath = EnableFastMath;
1824
1825 setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
1826 setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
1827 setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
1828 setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
1829 setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
1830 setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
1831 setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
1832 setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
1833
1834 setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
1835 setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
1836 setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
1837 setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
1838 setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
1839 setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
1840
1841 // This is the only fast library function for sqrtd.
1842 if (FastMath)
1843 setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
1844
1845 // Prefix is: nothing for "slow-math",
1846 // "fast2_" for V5+ fast-math double-precision
1847 // (actually, keep fast-math and fast-math2 separate for now)
1848 if (FastMath) {
1849 setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
1850 setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
1851 setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
1852 setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
1853 setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
1854 } else {
1855 setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
1856 setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
1857 setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
1858 setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
1859 setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
1860 }
1861
1862 if (FastMath)
1863 setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
1864 else
1865 setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
1866
1867 // Routines to handle fp16 storage type.
1868 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1869 setLibcallName(RTLIB::FPROUND_F64_F16, "__truncdfhf2");
1870 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1871
1872 // These cause problems when the shift amount is non-constant.
1873 setLibcallName(RTLIB::SHL_I128, nullptr);
1874 setLibcallName(RTLIB::SRL_I128, nullptr);
1875 setLibcallName(RTLIB::SRA_I128, nullptr);
1876}
1877
1878const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
1879 switch ((HexagonISD::NodeType)Opcode) {
1880 case HexagonISD::ADDC: return "HexagonISD::ADDC";
1881 case HexagonISD::SUBC: return "HexagonISD::SUBC";
1882 case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA";
1883 case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT";
1884 case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL";
1885 case HexagonISD::BARRIER: return "HexagonISD::BARRIER";
1886 case HexagonISD::CALL: return "HexagonISD::CALL";
1887 case HexagonISD::CALLnr: return "HexagonISD::CALLnr";
1888 case HexagonISD::CALLR: return "HexagonISD::CALLR";
1889 case HexagonISD::COMBINE: return "HexagonISD::COMBINE";
1890 case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP";
1891 case HexagonISD::CONST32: return "HexagonISD::CONST32";
1892 case HexagonISD::CP: return "HexagonISD::CP";
1893 case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH";
1894 case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN";
1895 case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT";
1896 case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU";
1897 case HexagonISD::INSERT: return "HexagonISD::INSERT";
1898 case HexagonISD::JT: return "HexagonISD::JT";
1899 case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG";
1900 case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN";
1901 case HexagonISD::VASL: return "HexagonISD::VASL";
1902 case HexagonISD::VASR: return "HexagonISD::VASR";
1903 case HexagonISD::VLSR: return "HexagonISD::VLSR";
1904 case HexagonISD::MFSHL: return "HexagonISD::MFSHL";
1905 case HexagonISD::MFSHR: return "HexagonISD::MFSHR";
1906 case HexagonISD::SSAT: return "HexagonISD::SSAT";
1907 case HexagonISD::USAT: return "HexagonISD::USAT";
1908 case HexagonISD::SMUL_LOHI: return "HexagonISD::SMUL_LOHI";
1909 case HexagonISD::UMUL_LOHI: return "HexagonISD::UMUL_LOHI";
1910 case HexagonISD::USMUL_LOHI: return "HexagonISD::USMUL_LOHI";
1911 case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW";
1912 case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0";
1913 case HexagonISD::VROR: return "HexagonISD::VROR";
1914 case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE";
1915 case HexagonISD::PTRUE: return "HexagonISD::PTRUE";
1916 case HexagonISD::PFALSE: return "HexagonISD::PFALSE";
1917 case HexagonISD::D2P: return "HexagonISD::D2P";
1918 case HexagonISD::P2D: return "HexagonISD::P2D";
1919 case HexagonISD::V2Q: return "HexagonISD::V2Q";
1920 case HexagonISD::Q2V: return "HexagonISD::Q2V";
1921 case HexagonISD::QCAT: return "HexagonISD::QCAT";
1922 case HexagonISD::QTRUE: return "HexagonISD::QTRUE";
1923 case HexagonISD::QFALSE: return "HexagonISD::QFALSE";
1924 case HexagonISD::TL_EXTEND: return "HexagonISD::TL_EXTEND";
1925 case HexagonISD::TL_TRUNCATE: return "HexagonISD::TL_TRUNCATE";
1926 case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST";
1927 case HexagonISD::VALIGN: return "HexagonISD::VALIGN";
1928 case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR";
1929 case HexagonISD::ISEL: return "HexagonISD::ISEL";
1930 case HexagonISD::OP_END: break;
1931 }
1932 return nullptr;
1933}
1934
1935bool
1936HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1937 const SDLoc &dl, SelectionDAG &DAG) const {
1938 auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1939 if (!CA)
1940 return true;
1941 unsigned Addr = CA->getZExtValue();
1942 Align HaveAlign =
1943 Addr != 0 ? Align(1ull << llvm::countr_zero(Addr)) : NeedAlign;
1944 if (HaveAlign >= NeedAlign)
1945 return true;
1946
1947 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1948
1949 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1950 DiagnosticInfoMisalignedTrap(StringRef M)
1951 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1952 void print(DiagnosticPrinter &DP) const override {
1953 DP << Msg;
1954 }
1955 static bool classof(const DiagnosticInfo *DI) {
1956 return DI->getKind() == DK_MisalignedTrap;
1957 }
1958 StringRef Msg;
1959 };
1960
1961 std::string ErrMsg;
1962 raw_string_ostream O(ErrMsg);
1963 O << "Misaligned constant address: " << format_hex(Addr, 10)
1964 << " has alignment " << HaveAlign.value()
1965 << ", but the memory access requires " << NeedAlign.value();
1966 if (DebugLoc DL = dl.getDebugLoc())
1967 DL.print(O << ", at ");
1968 O << ". The instruction has been replaced with a trap.";
1969
1970 DAG.getContext()->diagnose(DiagnosticInfoMisalignedTrap(O.str()));
1971 return false;
1972}
1973
1974SDValue
1975HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1976 const {
1977 const SDLoc &dl(Op);
1978 auto *LS = cast<LSBaseSDNode>(Op.getNode());
1979 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
1980
1981 SDValue Chain = LS->getChain();
1982 SDValue Trap = DAG.getNode(ISD::TRAP, dl, MVT::Other, Chain);
1983 if (LS->getOpcode() == ISD::LOAD)
1984 return DAG.getMergeValues({DAG.getUNDEF(ty(Op)), Trap}, dl);
1985 return Trap;
1986}
1987
1988// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1989// intrinsic.
1990static bool isBrevLdIntrinsic(const Value *Inst) {
1991 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1992 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1993 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1994 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1995 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1996 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1997 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1998}
1999
2000// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
2001// instruction. So far we only handle bitcast, extract value and bit reverse
2002// load intrinsic instructions. Should we handle CGEP ?
2004 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
2005 Operator::getOpcode(V) == Instruction::BitCast)
2006 V = cast<Operator>(V)->getOperand(0);
2007 else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V))
2008 V = cast<Instruction>(V)->getOperand(0);
2009 return V;
2010}
2011
2012// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
2013// a back edge. If the back edge comes from the intrinsic itself, the incoming
2014// edge is returned.
2015static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
2016 const BasicBlock *Parent = PN->getParent();
2017 int Idx = -1;
2018 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
2019 BasicBlock *Blk = PN->getIncomingBlock(i);
2020 // Determine if the back edge is originated from intrinsic.
2021 if (Blk == Parent) {
2022 Value *BackEdgeVal = PN->getIncomingValue(i);
2023 Value *BaseVal;
2024 // Loop over till we return the same Value or we hit the IntrBaseVal.
2025 do {
2026 BaseVal = BackEdgeVal;
2027 BackEdgeVal = getBrevLdObject(BackEdgeVal);
2028 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2029 // If the getBrevLdObject returns IntrBaseVal, we should return the
2030 // incoming edge.
2031 if (IntrBaseVal == BackEdgeVal)
2032 continue;
2033 Idx = i;
2034 break;
2035 } else // Set the node to incoming edge.
2036 Idx = i;
2037 }
2038 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
2039 return PN->getIncomingValue(Idx);
2040}
2041
2042// Bit-reverse Load Intrinsic: Figure out the underlying object the base
2043// pointer points to, for the bit-reverse load intrinsic. Setting this to
2044// memoperand might help alias analysis to figure out the dependencies.
2046 Value *IntrBaseVal = V;
2047 Value *BaseVal;
2048 // Loop over till we return the same Value, implies we either figure out
2049 // the object or we hit a PHI
2050 do {
2051 BaseVal = V;
2052 V = getBrevLdObject(V);
2053 } while (BaseVal != V);
2054
2055 // Identify the object from PHINode.
2056 if (const PHINode *PN = dyn_cast<PHINode>(V))
2057 return returnEdge(PN, IntrBaseVal);
2058 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2059 else
2060 return V;
2061}
2062
2063/// Given an intrinsic, checks if on the target the intrinsic will need to map
2064/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
2065/// true and store the intrinsic information into the IntrinsicInfo that was
2066/// passed to the function.
2068 const CallInst &I,
2069 MachineFunction &MF,
2070 unsigned Intrinsic) const {
2071 switch (Intrinsic) {
2072 case Intrinsic::hexagon_L2_loadrd_pbr:
2073 case Intrinsic::hexagon_L2_loadri_pbr:
2074 case Intrinsic::hexagon_L2_loadrh_pbr:
2075 case Intrinsic::hexagon_L2_loadruh_pbr:
2076 case Intrinsic::hexagon_L2_loadrb_pbr:
2077 case Intrinsic::hexagon_L2_loadrub_pbr: {
2079 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
2080 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2081 // The intrinsic function call is of the form { ElTy, i8* }
2082 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2083 // should be derived from ElTy.
2084 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);
2085 Info.memVT = MVT::getVT(ElTy);
2086 llvm::Value *BasePtrVal = I.getOperand(0);
2087 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);
2088 // The offset value comes through Modifier register. For now, assume the
2089 // offset is 0.
2090 Info.offset = 0;
2091 Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));
2093 return true;
2094 }
2095 case Intrinsic::hexagon_V6_vgathermw:
2096 case Intrinsic::hexagon_V6_vgathermw_128B:
2097 case Intrinsic::hexagon_V6_vgathermh:
2098 case Intrinsic::hexagon_V6_vgathermh_128B:
2099 case Intrinsic::hexagon_V6_vgathermhw:
2100 case Intrinsic::hexagon_V6_vgathermhw_128B:
2101 case Intrinsic::hexagon_V6_vgathermwq:
2102 case Intrinsic::hexagon_V6_vgathermwq_128B:
2103 case Intrinsic::hexagon_V6_vgathermhq:
2104 case Intrinsic::hexagon_V6_vgathermhq_128B:
2105 case Intrinsic::hexagon_V6_vgathermhwq:
2106 case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2107 const Module &M = *I.getParent()->getParent()->getParent();
2109 Type *VecTy = I.getArgOperand(1)->getType();
2110 Info.memVT = MVT::getVT(VecTy);
2111 Info.ptrVal = I.getArgOperand(0);
2112 Info.offset = 0;
2113 Info.align =
2114 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2118 return true;
2119 }
2120 default:
2121 break;
2122 }
2123 return false;
2124}
2125
2127 return X.getValueType().isScalarInteger(); // 'tstbit'
2128}
2129
2131 return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2));
2132}
2133
2135 if (!VT1.isSimple() || !VT2.isSimple())
2136 return false;
2137 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2138}
2139
2141 const MachineFunction &MF, EVT VT) const {
2143}
2144
2145// Should we expand the build vector with shuffles?
2147 unsigned DefinedValues) const {
2148 return false;
2149}
2150
2152 unsigned Index) const {
2154 if (!ResVT.isSimple() || !SrcVT.isSimple())
2155 return false;
2156
2157 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2158 if (ResTy.getVectorElementType() != MVT::i1)
2159 return true;
2160
2161 // Non-HVX bool vectors are relatively cheap.
2162 return SrcTy.getVectorNumElements() <= 8;
2163}
2164
2166 return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2168}
2169
2171 EVT VT) const {
2172 return true;
2173}
2174
2177 unsigned VecLen = VT.getVectorMinNumElements();
2178 MVT ElemTy = VT.getVectorElementType();
2179
2180 if (VecLen == 1 || VT.isScalableVector())
2182
2183 if (Subtarget.useHVXOps()) {
2184 unsigned Action = getPreferredHvxVectorAction(VT);
2185 if (Action != ~0u)
2186 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2187 }
2188
2189 // Always widen (remaining) vectors of i1.
2190 if (ElemTy == MVT::i1)
2192 // Widen non-power-of-2 vectors. Such types cannot be split right now,
2193 // and computeRegisterProperties will override "split" with "widen",
2194 // which can cause other issues.
2195 if (!isPowerOf2_32(VecLen))
2197
2199}
2200
2203 if (Subtarget.useHVXOps()) {
2204 unsigned Action = getCustomHvxOperationAction(Op);
2205 if (Action != ~0u)
2206 return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2207 }
2209}
2210
2211std::pair<SDValue, int>
2212HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2213 if (Addr.getOpcode() == ISD::ADD) {
2214 SDValue Op1 = Addr.getOperand(1);
2215 if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode()))
2216 return { Addr.getOperand(0), CN->getSExtValue() };
2217 }
2218 return { Addr, 0 };
2219}
2220
2221// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2222// to select data from, V3 is the permutation.
2223SDValue
2225 const {
2226 const auto *SVN = cast<ShuffleVectorSDNode>(Op);
2227 ArrayRef<int> AM = SVN->getMask();
2228 assert(AM.size() <= 8 && "Unexpected shuffle mask");
2229 unsigned VecLen = AM.size();
2230
2231 MVT VecTy = ty(Op);
2232 assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2233 "HVX shuffles should be legal");
2234 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2235
2236 SDValue Op0 = Op.getOperand(0);
2237 SDValue Op1 = Op.getOperand(1);
2238 const SDLoc &dl(Op);
2239
2240 // If the inputs are not the same as the output, bail. This is not an
2241 // error situation, but complicates the handling and the default expansion
2242 // (into BUILD_VECTOR) should be adequate.
2243 if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2244 return SDValue();
2245
2246 // Normalize the mask so that the first non-negative index comes from
2247 // the first operand.
2248 SmallVector<int,8> Mask(AM.begin(), AM.end());
2249 unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data();
2250 if (F == AM.size())
2251 return DAG.getUNDEF(VecTy);
2252 if (AM[F] >= int(VecLen)) {
2254 std::swap(Op0, Op1);
2255 }
2256
2257 // Express the shuffle mask in terms of bytes.
2258 SmallVector<int,8> ByteMask;
2259 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2260 for (int M : Mask) {
2261 if (M < 0) {
2262 for (unsigned j = 0; j != ElemBytes; ++j)
2263 ByteMask.push_back(-1);
2264 } else {
2265 for (unsigned j = 0; j != ElemBytes; ++j)
2266 ByteMask.push_back(M*ElemBytes + j);
2267 }
2268 }
2269 assert(ByteMask.size() <= 8);
2270
2271 // All non-undef (non-negative) indexes are well within [0..127], so they
2272 // fit in a single byte. Build two 64-bit words:
2273 // - MaskIdx where each byte is the corresponding index (for non-negative
2274 // indexes), and 0xFF for negative indexes, and
2275 // - MaskUnd that has 0xFF for each negative index.
2276 uint64_t MaskIdx = 0;
2277 uint64_t MaskUnd = 0;
2278 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2279 unsigned S = 8*i;
2280 uint64_t M = ByteMask[i] & 0xFF;
2281 if (M == 0xFF)
2282 MaskUnd |= M << S;
2283 MaskIdx |= M << S;
2284 }
2285
2286 if (ByteMask.size() == 4) {
2287 // Identity.
2288 if (MaskIdx == (0x03020100 | MaskUnd))
2289 return Op0;
2290 // Byte swap.
2291 if (MaskIdx == (0x00010203 | MaskUnd)) {
2292 SDValue T0 = DAG.getBitcast(MVT::i32, Op0);
2293 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);
2294 return DAG.getBitcast(VecTy, T1);
2295 }
2296
2297 // Byte packs.
2298 SDValue Concat10 =
2299 getCombine(Op1, Op0, dl, typeJoin({ty(Op1), ty(Op0)}), DAG);
2300 if (MaskIdx == (0x06040200 | MaskUnd))
2301 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2302 if (MaskIdx == (0x07050301 | MaskUnd))
2303 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2304
2305 SDValue Concat01 =
2306 getCombine(Op0, Op1, dl, typeJoin({ty(Op0), ty(Op1)}), DAG);
2307 if (MaskIdx == (0x02000604 | MaskUnd))
2308 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2309 if (MaskIdx == (0x03010705 | MaskUnd))
2310 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2311 }
2312
2313 if (ByteMask.size() == 8) {
2314 // Identity.
2315 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2316 return Op0;
2317 // Byte swap.
2318 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2319 SDValue T0 = DAG.getBitcast(MVT::i64, Op0);
2320 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);
2321 return DAG.getBitcast(VecTy, T1);
2322 }
2323
2324 // Halfword picks.
2325 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2326 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2327 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2328 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2329 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2330 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2331 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2332 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2333 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2334 VectorPair P = opSplit(Op0, dl, DAG);
2335 return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG);
2336 }
2337
2338 // Byte packs.
2339 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2340 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2341 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2342 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2343 }
2344
2345 return SDValue();
2346}
2347
2348SDValue
2349HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2350 switch (Op.getOpcode()) {
2351 case ISD::BUILD_VECTOR:
2352 if (SDValue S = cast<BuildVectorSDNode>(Op)->getSplatValue())
2353 return S;
2354 break;
2355 case ISD::SPLAT_VECTOR:
2356 return Op.getOperand(0);
2357 }
2358 return SDValue();
2359}
2360
2361// Create a Hexagon-specific node for shifting a vector by an integer.
2362SDValue
2363HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2364 const {
2365 unsigned NewOpc;
2366 switch (Op.getOpcode()) {
2367 case ISD::SHL:
2368 NewOpc = HexagonISD::VASL;
2369 break;
2370 case ISD::SRA:
2371 NewOpc = HexagonISD::VASR;
2372 break;
2373 case ISD::SRL:
2374 NewOpc = HexagonISD::VLSR;
2375 break;
2376 default:
2377 llvm_unreachable("Unexpected shift opcode");
2378 }
2379
2380 if (SDValue Sp = getSplatValue(Op.getOperand(1), DAG))
2381 return DAG.getNode(NewOpc, SDLoc(Op), ty(Op), Op.getOperand(0), Sp);
2382 return SDValue();
2383}
2384
2385SDValue
2387 const SDLoc &dl(Op);
2388
2389 // First try to convert the shift (by vector) to a shift by a scalar.
2390 // If we first split the shift, the shift amount will become 'extract
2391 // subvector', and will no longer be recognized as scalar.
2392 SDValue Res = Op;
2393 if (SDValue S = getVectorShiftByInt(Op, DAG))
2394 Res = S;
2395
2396 unsigned Opc = Res.getOpcode();
2397 switch (Opc) {
2398 case HexagonISD::VASR:
2399 case HexagonISD::VLSR:
2400 case HexagonISD::VASL:
2401 break;
2402 default:
2403 // No instructions for shifts by non-scalars.
2404 return SDValue();
2405 }
2406
2407 MVT ResTy = ty(Res);
2408 if (ResTy.getVectorElementType() != MVT::i8)
2409 return Res;
2410
2411 // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2413 SDValue Val = Res.getOperand(0), Amt = Res.getOperand(1);
2414
2415 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2416 MVT Ty = ty(V);
2418 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(V, dl, ExtTy)
2419 : DAG.getZExtOrTrunc(V, dl, ExtTy);
2420 SDValue ExtS = DAG.getNode(Opc, dl, ExtTy, {ExtV, A});
2421 return DAG.getZExtOrTrunc(ExtS, dl, Ty);
2422 };
2423
2424 if (ResTy.getSizeInBits() == 32)
2425 return ShiftPartI8(Opc, Val, Amt);
2426
2427 auto [LoV, HiV] = opSplit(Val, dl, DAG);
2428 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy,
2429 {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2430}
2431
2432SDValue
2434 if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))
2435 return Op;
2436 return SDValue();
2437}
2438
2439SDValue
2441 MVT ResTy = ty(Op);
2442 SDValue InpV = Op.getOperand(0);
2443 MVT InpTy = ty(InpV);
2444 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2445 const SDLoc &dl(Op);
2446
2447 // Handle conversion from i8 to v8i1.
2448 if (InpTy == MVT::i8) {
2449 if (ResTy == MVT::v8i1) {
2450 SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);
2451 SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);
2452 return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);
2453 }
2454 return SDValue();
2455 }
2456
2457 return Op;
2458}
2459
2460bool
2461HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2462 MVT VecTy, SelectionDAG &DAG,
2463 MutableArrayRef<ConstantInt*> Consts) const {
2464 MVT ElemTy = VecTy.getVectorElementType();
2465 unsigned ElemWidth = ElemTy.getSizeInBits();
2466 IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth);
2467 bool AllConst = true;
2468
2469 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2470 SDValue V = Values[i];
2471 if (V.isUndef()) {
2472 Consts[i] = ConstantInt::get(IntTy, 0);
2473 continue;
2474 }
2475 // Make sure to always cast to IntTy.
2476 if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {
2477 const ConstantInt *CI = CN->getConstantIntValue();
2478 Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue());
2479 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) {
2480 const ConstantFP *CF = CN->getConstantFPValue();
2482 Consts[i] = ConstantInt::get(IntTy, A.getZExtValue());
2483 } else {
2484 AllConst = false;
2485 }
2486 }
2487 return AllConst;
2488}
2489
2490SDValue
2491HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2492 MVT VecTy, SelectionDAG &DAG) const {
2493 MVT ElemTy = VecTy.getVectorElementType();
2494 assert(VecTy.getVectorNumElements() == Elem.size());
2495
2496 SmallVector<ConstantInt*,4> Consts(Elem.size());
2497 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2498
2499 unsigned First, Num = Elem.size();
2500 for (First = 0; First != Num; ++First) {
2501 if (!isUndef(Elem[First]))
2502 break;
2503 }
2504 if (First == Num)
2505 return DAG.getUNDEF(VecTy);
2506
2507 if (AllConst &&
2508 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2509 return getZero(dl, VecTy, DAG);
2510
2511 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2512 assert(Elem.size() == 2);
2513 if (AllConst) {
2514 // The 'Consts' array will have all values as integers regardless
2515 // of the vector element type.
2516 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2517 Consts[1]->getZExtValue() << 16;
2518 return DAG.getBitcast(VecTy, DAG.getConstant(V, dl, MVT::i32));
2519 }
2520 SDValue E0, E1;
2521 if (ElemTy == MVT::f16) {
2522 E0 = DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Elem[0]), dl, MVT::i32);
2523 E1 = DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Elem[1]), dl, MVT::i32);
2524 } else {
2525 E0 = Elem[0];
2526 E1 = Elem[1];
2527 }
2528 SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {E1, E0}, DAG);
2529 return DAG.getBitcast(VecTy, N);
2530 }
2531
2532 if (ElemTy == MVT::i8) {
2533 // First try generating a constant.
2534 if (AllConst) {
2535 int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2536 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2537 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2538 Consts[3]->getZExtValue() << 24;
2539 return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32));
2540 }
2541
2542 // Then try splat.
2543 bool IsSplat = true;
2544 for (unsigned i = First+1; i != Num; ++i) {
2545 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2546 continue;
2547 IsSplat = false;
2548 break;
2549 }
2550 if (IsSplat) {
2551 // Legalize the operand of SPLAT_VECTOR.
2552 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2553 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2554 }
2555
2556 // Generate
2557 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2558 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2559 assert(Elem.size() == 4);
2560 SDValue Vs[4];
2561 for (unsigned i = 0; i != 4; ++i) {
2562 Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32);
2563 Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8);
2564 }
2565 SDValue S8 = DAG.getConstant(8, dl, MVT::i32);
2566 SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});
2567 SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});
2568 SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});
2569 SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});
2570
2571 SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);
2572 return DAG.getBitcast(MVT::v4i8, R);
2573 }
2574
2575#ifndef NDEBUG
2576 dbgs() << "VecTy: " << VecTy << '\n';
2577#endif
2578 llvm_unreachable("Unexpected vector element type");
2579}
2580
2581SDValue
2582HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2583 MVT VecTy, SelectionDAG &DAG) const {
2584 MVT ElemTy = VecTy.getVectorElementType();
2585 assert(VecTy.getVectorNumElements() == Elem.size());
2586
2587 SmallVector<ConstantInt*,8> Consts(Elem.size());
2588 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2589
2590 unsigned First, Num = Elem.size();
2591 for (First = 0; First != Num; ++First) {
2592 if (!isUndef(Elem[First]))
2593 break;
2594 }
2595 if (First == Num)
2596 return DAG.getUNDEF(VecTy);
2597
2598 if (AllConst &&
2599 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2600 return getZero(dl, VecTy, DAG);
2601
2602 // First try splat if possible.
2603 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2604 bool IsSplat = true;
2605 for (unsigned i = First+1; i != Num; ++i) {
2606 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2607 continue;
2608 IsSplat = false;
2609 break;
2610 }
2611 if (IsSplat) {
2612 // Legalize the operand of SPLAT_VECTOR
2613 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(MVT::i16, Elem[First])
2614 : Elem[First];
2615 SDValue Ext = DAG.getZExtOrTrunc(S, dl, MVT::i32);
2616 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2617 }
2618 }
2619
2620 // Then try constant.
2621 if (AllConst) {
2622 uint64_t Val = 0;
2623 unsigned W = ElemTy.getSizeInBits();
2624 uint64_t Mask = (1ull << W) - 1;
2625 for (unsigned i = 0; i != Num; ++i)
2626 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2627 SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);
2628 return DAG.getBitcast(VecTy, V0);
2629 }
2630
2631 // Build two 32-bit vectors and concatenate.
2632 MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2);
2633 SDValue L = (ElemTy == MVT::i32)
2634 ? Elem[0]
2635 : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG);
2636 SDValue H = (ElemTy == MVT::i32)
2637 ? Elem[1]
2638 : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG);
2639 return getCombine(H, L, dl, VecTy, DAG);
2640}
2641
2642SDValue
2643HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2644 const SDLoc &dl, MVT ValTy, MVT ResTy,
2645 SelectionDAG &DAG) const {
2646 MVT VecTy = ty(VecV);
2647 assert(!ValTy.isVector() ||
2648 VecTy.getVectorElementType() == ValTy.getVectorElementType());
2649 if (VecTy.getVectorElementType() == MVT::i1)
2650 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2651
2652 unsigned VecWidth = VecTy.getSizeInBits();
2653 unsigned ValWidth = ValTy.getSizeInBits();
2654 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2655 assert((VecWidth % ElemWidth) == 0);
2656 assert(VecWidth == 32 || VecWidth == 64);
2657
2658 // Cast everything to scalar integer types.
2659 MVT ScalarTy = tyScalar(VecTy);
2660 VecV = DAG.getBitcast(ScalarTy, VecV);
2661
2662 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2663 SDValue ExtV;
2664
2665 if (auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {
2666 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2667 if (VecWidth == 64 && ValWidth == 32) {
2668 assert(Off == 0 || Off == 32);
2669 ExtV = Off == 0 ? LoHalf(VecV, DAG) : HiHalf(VecV, DAG);
2670 } else if (Off == 0 && (ValWidth % 8) == 0) {
2671 ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy));
2672 } else {
2673 SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);
2674 // The return type of EXTRACTU must be the same as the type of the
2675 // input vector.
2676 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2677 {VecV, WidthV, OffV});
2678 }
2679 } else {
2680 if (ty(IdxV) != MVT::i32)
2681 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2682 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2683 DAG.getConstant(ElemWidth, dl, MVT::i32));
2684 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2685 {VecV, WidthV, OffV});
2686 }
2687
2688 // Cast ExtV to the requested result type.
2689 ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy));
2690 ExtV = DAG.getBitcast(ResTy, ExtV);
2691 return ExtV;
2692}
2693
2694SDValue
2695HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2696 const SDLoc &dl, MVT ValTy, MVT ResTy,
2697 SelectionDAG &DAG) const {
2698 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2699 // without any coprocessors).
2700 MVT VecTy = ty(VecV);
2701 unsigned VecWidth = VecTy.getSizeInBits();
2702 unsigned ValWidth = ValTy.getSizeInBits();
2703 assert(VecWidth == VecTy.getVectorNumElements() &&
2704 "Vector elements should equal vector width size");
2705 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2706
2707 // Check if this is an extract of the lowest bit.
2708 if (auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {
2709 // Extracting the lowest bit is a no-op, but it changes the type,
2710 // so it must be kept as an operation to avoid errors related to
2711 // type mismatches.
2712 if (IdxN->isZero() && ValTy.getSizeInBits() == 1)
2713 return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV);
2714 }
2715
2716 // If the value extracted is a single bit, use tstbit.
2717 if (ValWidth == 1) {
2718 SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2719 SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);
2720 SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);
2721 return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0);
2722 }
2723
2724 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2725 // a predicate register. The elements of the vector are repeated
2726 // in the register (if necessary) so that the total number is 8.
2727 // The extracted subvector will need to be expanded in such a way.
2728 unsigned Scale = VecWidth / ValWidth;
2729
2730 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2731 // position 0.
2732 assert(ty(IdxV) == MVT::i32);
2733 unsigned VecRep = 8 / VecWidth;
2734 SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2735 DAG.getConstant(8*VecRep, dl, MVT::i32));
2736 SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2737 SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);
2738 while (Scale > 1) {
2739 // The longest possible subvector is at most 32 bits, so it is always
2740 // contained in the low subregister.
2741 T1 = LoHalf(T1, DAG);
2742 T1 = expandPredicate(T1, dl, DAG);
2743 Scale /= 2;
2744 }
2745
2746 return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1);
2747}
2748
2749SDValue
2750HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2751 const SDLoc &dl, MVT ValTy,
2752 SelectionDAG &DAG) const {
2753 MVT VecTy = ty(VecV);
2754 if (VecTy.getVectorElementType() == MVT::i1)
2755 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2756
2757 unsigned VecWidth = VecTy.getSizeInBits();
2758 unsigned ValWidth = ValTy.getSizeInBits();
2759 assert(VecWidth == 32 || VecWidth == 64);
2760 assert((VecWidth % ValWidth) == 0);
2761
2762 // Cast everything to scalar integer types.
2763 MVT ScalarTy = MVT::getIntegerVT(VecWidth);
2764 // The actual type of ValV may be different than ValTy (which is related
2765 // to the vector type).
2766 unsigned VW = ty(ValV).getSizeInBits();
2767 ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV);
2768 VecV = DAG.getBitcast(ScalarTy, VecV);
2769 if (VW != VecWidth)
2770 ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy);
2771
2772 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2773 SDValue InsV;
2774
2775 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) {
2776 unsigned W = C->getZExtValue() * ValWidth;
2777 SDValue OffV = DAG.getConstant(W, dl, MVT::i32);
2778 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2779 {VecV, ValV, WidthV, OffV});
2780 } else {
2781 if (ty(IdxV) != MVT::i32)
2782 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2783 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);
2784 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2785 {VecV, ValV, WidthV, OffV});
2786 }
2787
2788 return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV);
2789}
2790
2791SDValue
2792HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2793 SDValue IdxV, const SDLoc &dl,
2794 MVT ValTy, SelectionDAG &DAG) const {
2795 MVT VecTy = ty(VecV);
2796 unsigned VecLen = VecTy.getVectorNumElements();
2797
2798 if (ValTy == MVT::i1) {
2799 SDValue ToReg = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2800 SDValue Ext = DAG.getSExtOrTrunc(ValV, dl, MVT::i32);
2801 SDValue Width = DAG.getConstant(8 / VecLen, dl, MVT::i32);
2802 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, Width);
2803 SDValue Ins =
2804 DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, {ToReg, Ext, Width, Idx});
2805 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Ins}, DAG);
2806 }
2807
2809 SDValue ValR = ValTy.isVector()
2810 ? DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV)
2811 : DAG.getSExtOrTrunc(ValV, dl, MVT::i64);
2812
2813 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2814 assert(Scale > 1);
2815
2816 for (unsigned R = Scale; R > 1; R /= 2) {
2817 ValR = contractPredicate(ValR, dl, DAG);
2818 ValR = getCombine(DAG.getUNDEF(MVT::i32), ValR, dl, MVT::i64, DAG);
2819 }
2820
2821 SDValue Width = DAG.getConstant(64 / Scale, dl, MVT::i32);
2822 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, Width);
2823 SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2824 SDValue Ins =
2825 DAG.getNode(HexagonISD::INSERT, dl, MVT::i64, {VecR, ValR, Width, Idx});
2826 return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins);
2827}
2828
2829SDValue
2830HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2831 SelectionDAG &DAG) const {
2832 assert(ty(Vec32).getSizeInBits() == 32);
2833 if (isUndef(Vec32))
2834 return DAG.getUNDEF(MVT::i64);
2835 SDValue P = DAG.getBitcast(MVT::v4i8, Vec32);
2837 return DAG.getBitcast(MVT::i64, X);
2838}
2839
2840SDValue
2841HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2842 SelectionDAG &DAG) const {
2843 assert(ty(Vec64).getSizeInBits() == 64);
2844 if (isUndef(Vec64))
2845 return DAG.getUNDEF(MVT::i32);
2846 // Collect even bytes:
2847 SDValue A = DAG.getBitcast(MVT::v8i8, Vec64);
2849 {0, 2, 4, 6, 1, 3, 5, 7});
2850 return extractVector(S, DAG.getConstant(0, dl, MVT::i32), dl, MVT::v4i8,
2851 MVT::i32, DAG);
2852}
2853
2854SDValue
2855HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2856 const {
2857 if (Ty.isVector()) {
2858 unsigned W = Ty.getSizeInBits();
2859 if (W <= 64)
2860 return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W)));
2861 return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG));
2862 }
2863
2864 if (Ty.isInteger())
2865 return DAG.getConstant(0, dl, Ty);
2866 if (Ty.isFloatingPoint())
2867 return DAG.getConstantFP(0.0, dl, Ty);
2868 llvm_unreachable("Invalid type for zero");
2869}
2870
2871SDValue
2872HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2873 const {
2874 MVT ValTy = ty(Val);
2876
2877 unsigned ValLen = ValTy.getVectorNumElements();
2878 unsigned ResLen = ResTy.getVectorNumElements();
2879 if (ValLen == ResLen)
2880 return Val;
2881
2882 const SDLoc &dl(Val);
2883 assert(ValLen < ResLen);
2884 assert(ResLen % ValLen == 0);
2885
2886 SmallVector<SDValue, 4> Concats = {Val};
2887 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2888 Concats.push_back(DAG.getUNDEF(ValTy));
2889
2890 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats);
2891}
2892
2893SDValue
2894HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
2895 MVT ResTy, SelectionDAG &DAG) const {
2896 MVT ElemTy = ty(Hi);
2897 assert(ElemTy == ty(Lo));
2898
2899 if (!ElemTy.isVector()) {
2900 assert(ElemTy.isScalarInteger());
2901 MVT PairTy = MVT::getIntegerVT(2 * ElemTy.getSizeInBits());
2902 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, PairTy, Lo, Hi);
2903 return DAG.getBitcast(ResTy, Pair);
2904 }
2905
2906 unsigned Width = ElemTy.getSizeInBits();
2907 MVT IntTy = MVT::getIntegerVT(Width);
2908 MVT PairTy = MVT::getIntegerVT(2 * Width);
2909 SDValue Pair =
2911 {DAG.getBitcast(IntTy, Lo), DAG.getBitcast(IntTy, Hi)});
2912 return DAG.getBitcast(ResTy, Pair);
2913}
2914
2915SDValue
2917 MVT VecTy = ty(Op);
2918 unsigned BW = VecTy.getSizeInBits();
2919 const SDLoc &dl(Op);
2921 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2922 Ops.push_back(Op.getOperand(i));
2923
2924 if (BW == 32)
2925 return buildVector32(Ops, dl, VecTy, DAG);
2926 if (BW == 64)
2927 return buildVector64(Ops, dl, VecTy, DAG);
2928
2929 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2930 // Check if this is a special case or all-0 or all-1.
2931 bool All0 = true, All1 = true;
2932 for (SDValue P : Ops) {
2933 auto *CN = dyn_cast<ConstantSDNode>(P.getNode());
2934 if (CN == nullptr) {
2935 All0 = All1 = false;
2936 break;
2937 }
2938 uint32_t C = CN->getZExtValue();
2939 All0 &= (C == 0);
2940 All1 &= (C == 1);
2941 }
2942 if (All0)
2943 return DAG.getNode(HexagonISD::PFALSE, dl, VecTy);
2944 if (All1)
2945 return DAG.getNode(HexagonISD::PTRUE, dl, VecTy);
2946
2947 // For each i1 element in the resulting predicate register, put 1
2948 // shifted by the index of the element into a general-purpose register,
2949 // then or them together and transfer it back into a predicate register.
2950 SDValue Rs[8];
2951 SDValue Z = getZero(dl, MVT::i32, DAG);
2952 // Always produce 8 bits, repeat inputs if necessary.
2953 unsigned Rep = 8 / VecTy.getVectorNumElements();
2954 for (unsigned i = 0; i != 8; ++i) {
2955 SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);
2956 Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z);
2957 }
2958 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {
2959 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2960 Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]);
2961 }
2962 // Move the value directly to a predicate register.
2963 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2964 }
2965
2966 return SDValue();
2967}
2968
2969SDValue
2971 SelectionDAG &DAG) const {
2972 MVT VecTy = ty(Op);
2973 const SDLoc &dl(Op);
2974 if (VecTy.getSizeInBits() == 64) {
2975 assert(Op.getNumOperands() == 2);
2976 return getCombine(Op.getOperand(1), Op.getOperand(0), dl, VecTy, DAG);
2977 }
2978
2979 MVT ElemTy = VecTy.getVectorElementType();
2980 if (ElemTy == MVT::i1) {
2981 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
2982 MVT OpTy = ty(Op.getOperand(0));
2983 // Scale is how many times the operands need to be contracted to match
2984 // the representation in the target register.
2985 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2986 assert(Scale == Op.getNumOperands() && Scale > 1);
2987
2988 // First, convert all bool vectors to integers, then generate pairwise
2989 // inserts to form values of doubled length. Up until there are only
2990 // two values left to concatenate, all of these values will fit in a
2991 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2992 SmallVector<SDValue,4> Words[2];
2993 unsigned IdxW = 0;
2994
2995 for (SDValue P : Op.getNode()->op_values()) {
2996 SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);
2997 for (unsigned R = Scale; R > 1; R /= 2) {
2998 W = contractPredicate(W, dl, DAG);
2999 W = getCombine(DAG.getUNDEF(MVT::i32), W, dl, MVT::i64, DAG);
3000 }
3001 W = LoHalf(W, DAG);
3002 Words[IdxW].push_back(W);
3003 }
3004
3005 while (Scale > 2) {
3006 SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);
3007 Words[IdxW ^ 1].clear();
3008
3009 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
3010 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
3011 // Insert W1 into W0 right next to the significant bits of W0.
3013 {W0, W1, WidthV, WidthV});
3014 Words[IdxW ^ 1].push_back(T);
3015 }
3016 IdxW ^= 1;
3017 Scale /= 2;
3018 }
3019
3020 // At this point there should only be two words left, and Scale should be 2.
3021 assert(Scale == 2 && Words[IdxW].size() == 2);
3022
3023 SDValue WW = getCombine(Words[IdxW][1], Words[IdxW][0], dl, MVT::i64, DAG);
3024 return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW);
3025 }
3026
3027 return SDValue();
3028}
3029
3030SDValue
3032 SelectionDAG &DAG) const {
3033 SDValue Vec = Op.getOperand(0);
3034 MVT ElemTy = ty(Vec).getVectorElementType();
3035 return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG);
3036}
3037
3038SDValue
3040 SelectionDAG &DAG) const {
3041 return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op),
3042 ty(Op), ty(Op), DAG);
3043}
3044
3045SDValue
3047 SelectionDAG &DAG) const {
3048 return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
3049 SDLoc(Op), ty(Op).getVectorElementType(), DAG);
3050}
3051
3052SDValue
3054 SelectionDAG &DAG) const {
3055 SDValue ValV = Op.getOperand(1);
3056 return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),
3057 SDLoc(Op), ty(ValV), DAG);
3058}
3059
3060bool
3062 // Assuming the caller does not have either a signext or zeroext modifier, and
3063 // only one value is accepted, any reasonable truncation is allowed.
3064 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3065 return false;
3066
3067 // FIXME: in principle up to 64-bit could be made safe, but it would be very
3068 // fragile at the moment: any support for multiple value returns would be
3069 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3070 return Ty1->getPrimitiveSizeInBits() <= 32;
3071}
3072
3073SDValue
3075 MVT Ty = ty(Op);
3076 const SDLoc &dl(Op);
3077 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
3078 MVT MemTy = LN->getMemoryVT().getSimpleVT();
3080
3081 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3082 if (LoadPred) {
3083 SDValue NL = DAG.getLoad(
3085 LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(),
3086 /*MemoryVT*/ MVT::i8, LN->getAlign(), LN->getMemOperand()->getFlags(),
3087 LN->getAAInfo(), LN->getRanges());
3088 LN = cast<LoadSDNode>(NL.getNode());
3089 }
3090
3091 Align ClaimAlign = LN->getAlign();
3092 if (!validateConstPtrAlignment(LN->getBasePtr(), ClaimAlign, dl, DAG))
3093 return replaceMemWithUndef(Op, DAG);
3094
3095 // Call LowerUnalignedLoad for all loads, it recognizes loads that
3096 // don't need extra aligning.
3097 SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG);
3098 if (LoadPred) {
3099 SDValue TP = getInstr(Hexagon::C2_tfrrp, dl, MemTy, {LU}, DAG);
3100 if (ET == ISD::SEXTLOAD) {
3101 TP = DAG.getSExtOrTrunc(TP, dl, Ty);
3102 } else if (ET != ISD::NON_EXTLOAD) {
3103 TP = DAG.getZExtOrTrunc(TP, dl, Ty);
3104 }
3105 SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain();
3106 return DAG.getMergeValues({TP, Ch}, dl);
3107 }
3108 return LU;
3109}
3110
3111SDValue
3113 const SDLoc &dl(Op);
3114 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
3115 SDValue Val = SN->getValue();
3116 MVT Ty = ty(Val);
3117
3118 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3119 // Store the exact predicate (all bits).
3120 SDValue TR = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {Val}, DAG);
3121 SDValue NS = DAG.getTruncStore(SN->getChain(), dl, TR, SN->getBasePtr(),
3122 MVT::i8, SN->getMemOperand());
3123 if (SN->isIndexed()) {
3124 NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(),
3125 SN->getAddressingMode());
3126 }
3127 SN = cast<StoreSDNode>(NS.getNode());
3128 }
3129
3130 Align ClaimAlign = SN->getAlign();
3131 if (!validateConstPtrAlignment(SN->getBasePtr(), ClaimAlign, dl, DAG))
3132 return replaceMemWithUndef(Op, DAG);
3133
3134 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3135 Align NeedAlign = Subtarget.getTypeAlignment(StoreTy);
3136 if (ClaimAlign < NeedAlign)
3137 return expandUnalignedStore(SN, DAG);
3138 return SDValue(SN, 0);
3139}
3140
3141SDValue
3143 const {
3144 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
3145 MVT LoadTy = ty(Op);
3146 unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy).value();
3147 unsigned HaveAlign = LN->getAlign().value();
3148 if (HaveAlign >= NeedAlign)
3149 return Op;
3150
3151 const SDLoc &dl(Op);
3152 const DataLayout &DL = DAG.getDataLayout();
3153 LLVMContext &Ctx = *DAG.getContext();
3154
3155 // If the load aligning is disabled or the load can be broken up into two
3156 // smaller legal loads, do the default (target-independent) expansion.
3157 bool DoDefault = false;
3158 // Handle it in the default way if this is an indexed load.
3159 if (!LN->isUnindexed())
3160 DoDefault = true;
3161
3162 if (!AlignLoads) {
3164 *LN->getMemOperand()))
3165 return Op;
3166 DoDefault = true;
3167 }
3168 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3169 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3170 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)
3171 : MVT::getVectorVT(MVT::i8, HaveAlign);
3172 DoDefault =
3173 allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());
3174 }
3175 if (DoDefault) {
3176 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
3177 return DAG.getMergeValues({P.first, P.second}, dl);
3178 }
3179
3180 // The code below generates two loads, both aligned as NeedAlign, and
3181 // with the distance of NeedAlign between them. For that to cover the
3182 // bits that need to be loaded (and without overlapping), the size of
3183 // the loads should be equal to NeedAlign. This is true for all loadable
3184 // types, but add an assertion in case something changes in the future.
3185 assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3186
3187 unsigned LoadLen = NeedAlign;
3188 SDValue Base = LN->getBasePtr();
3189 SDValue Chain = LN->getChain();
3190 auto BO = getBaseAndOffset(Base);
3191 unsigned BaseOpc = BO.first.getOpcode();
3192 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3193 return Op;
3194
3195 if (BO.second % LoadLen != 0) {
3196 BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first,
3197 DAG.getConstant(BO.second % LoadLen, dl, MVT::i32));
3198 BO.second -= BO.second % LoadLen;
3199 }
3200 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3201 ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first,
3202 DAG.getConstant(NeedAlign, dl, MVT::i32))
3203 : BO.first;
3204 SDValue Base0 =
3205 DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl);
3206 SDValue Base1 = DAG.getMemBasePlusOffset(
3207 BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl);
3208
3209 MachineMemOperand *WideMMO = nullptr;
3210 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3212 WideMMO = MF.getMachineMemOperand(
3213 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
3214 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
3215 MMO->getSuccessOrdering(), MMO->getFailureOrdering());
3216 }
3217
3218 SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
3219 SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);
3220
3221 SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,
3222 {Load1, Load0, BaseNoOff.getOperand(0)});
3223 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3224 Load0.getValue(1), Load1.getValue(1));
3225 SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);
3226 return M;
3227}
3228
3229SDValue
3231 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
3232 auto *CY = dyn_cast<ConstantSDNode>(Y);
3233 if (!CY)
3234 return SDValue();
3235
3236 const SDLoc &dl(Op);
3237 SDVTList VTs = Op.getNode()->getVTList();
3238 assert(VTs.NumVTs == 2);
3239 assert(VTs.VTs[1] == MVT::i1);
3240 unsigned Opc = Op.getOpcode();
3241
3242 if (CY) {
3243 uint32_t VY = CY->getZExtValue();
3244 assert(VY != 0 && "This should have been folded");
3245 // X +/- 1
3246 if (VY != 1)
3247 return SDValue();
3248
3249 if (Opc == ISD::UADDO) {
3250 SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});
3251 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),
3252 ISD::SETEQ);
3253 return DAG.getMergeValues({Op, Ov}, dl);
3254 }
3255 if (Opc == ISD::USUBO) {
3256 SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});
3257 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,
3258 DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ);
3259 return DAG.getMergeValues({Op, Ov}, dl);
3260 }
3261 }
3262
3263 return SDValue();
3264}
3265
3266SDValue
3268 const SDLoc &dl(Op);
3269 unsigned Opc = Op.getOpcode();
3270 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
3271
3272 if (Opc == ISD::ADDCARRY)
3273 return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),
3274 { X, Y, C });
3275
3276 EVT CarryTy = C.getValueType();
3277 SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),
3278 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3279 SDValue Out[] = { SubC.getValue(0),
3280 DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) };
3281 return DAG.getMergeValues(Out, dl);
3282}
3283
3284SDValue
3286 SDValue Chain = Op.getOperand(0);
3287 SDValue Offset = Op.getOperand(1);
3288 SDValue Handler = Op.getOperand(2);
3289 SDLoc dl(Op);
3290 auto PtrVT = getPointerTy(DAG.getDataLayout());
3291
3292 // Mark function as containing a call to EH_RETURN.
3296
3297 unsigned OffsetReg = Hexagon::R28;
3298
3299 SDValue StoreAddr =
3300 DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),
3301 DAG.getIntPtrConstant(4, dl));
3302 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
3303 Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
3304
3305 // Not needed we already use it as explict input to EH_RETURN.
3306 // MF.getRegInfo().addLiveOut(OffsetReg);
3307
3308 return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);
3309}
3310
3311SDValue
3313 unsigned Opc = Op.getOpcode();
3314
3315 // Handle INLINEASM first.
3316 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3317 return LowerINLINEASM(Op, DAG);
3318
3319 if (isHvxOperation(Op.getNode(), DAG)) {
3320 // If HVX lowering returns nothing, try the default lowering.
3321 if (SDValue V = LowerHvxOperation(Op, DAG))
3322 return V;
3323 }
3324
3325 switch (Opc) {
3326 default:
3327#ifndef NDEBUG
3328 Op.getNode()->dumpr(&DAG);
3329 if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)
3330 errs() << "Error: check for a non-legal type in this operation\n";
3331#endif
3332 llvm_unreachable("Should not custom lower this!");
3333 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3334 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3335 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3336 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3338 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3339 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3340 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3341 case ISD::LOAD: return LowerLoad(Op, DAG);
3342 case ISD::STORE: return LowerStore(Op, DAG);
3343 case ISD::UADDO:
3344 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3345 case ISD::ADDCARRY:
3346 case ISD::SUBCARRY: return LowerAddSubCarry(Op, DAG);
3347 case ISD::SRA:
3348 case ISD::SHL:
3349 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3350 case ISD::ROTL: return LowerROTL(Op, DAG);
3351 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3352 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3353 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3354 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3355 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3356 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3357 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3358 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3359 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3361 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3362 case ISD::VASTART: return LowerVASTART(Op, DAG);
3364 case ISD::SETCC: return LowerSETCC(Op, DAG);
3365 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3367 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3368 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG);
3369 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG);
3370 break;
3371 }
3372
3373 return SDValue();
3374}
3375
3376void
3379 SelectionDAG &DAG) const {
3380 if (isHvxOperation(N, DAG)) {
3381 LowerHvxOperationWrapper(N, Results, DAG);
3382 if (!Results.empty())
3383 return;
3384 }
3385
3386 SDValue Op(N, 0);
3387 unsigned Opc = N->getOpcode();
3388
3389 switch (Opc) {
3390 case HexagonISD::SSAT:
3391 case HexagonISD::USAT:
3392 Results.push_back(opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG));
3393 break;
3394 case ISD::STORE:
3395 // We are only custom-lowering stores to verify the alignment of the
3396 // address if it is a compile-time constant. Since a store can be
3397 // modified during type-legalization (the value being stored may need
3398 // legalization), return empty Results here to indicate that we don't
3399 // really make any changes in the custom lowering.
3400 return;
3401 default:
3403 break;
3404 }
3405}
3406
3407void
3410 SelectionDAG &DAG) const {
3411 if (isHvxOperation(N, DAG)) {
3412 ReplaceHvxNodeResults(N, Results, DAG);
3413 if (!Results.empty())
3414 return;
3415 }
3416
3417 const SDLoc &dl(N);
3418 switch (N->getOpcode()) {
3419 case ISD::SRL:
3420 case ISD::SRA:
3421 case ISD::SHL:
3422 return;
3423 case ISD::BITCAST:
3424 // Handle a bitcast from v8i1 to i8.
3425 if (N->getValueType(0) == MVT::i8) {
3426 if (N->getOperand(0).getValueType() == MVT::v8i1) {
3427 SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,
3428 N->getOperand(0), DAG);
3429 SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);
3430 Results.push_back(T);
3431 }
3432 }
3433 break;
3434 }
3435}
3436
3437SDValue
3439 DAGCombinerInfo &DCI) const {
3440 if (isHvxOperation(N, DCI.DAG)) {
3441 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3442 return V;
3443 return SDValue();
3444 }
3445
3446 SDValue Op(N, 0);
3447 const SDLoc &dl(Op);
3448 unsigned Opc = Op.getOpcode();
3449
3450 if (Opc == ISD::TRUNCATE) {
3451 SDValue Op0 = Op.getOperand(0);
3452 // fold (truncate (build pair x, y)) -> (truncate x) or x
3453 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3454 EVT TruncTy = Op.getValueType();
3455 SDValue Elem0 = Op0.getOperand(0);
3456 // if we match the low element of the pair, just return it.
3457 if (Elem0.getValueType() == TruncTy)
3458 return Elem0;
3459 // otherwise, if the low part is still too large, apply the truncate.
3460 if (Elem0.getValueType().bitsGT(TruncTy))
3461 return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);
3462 }
3463 }
3464
3465 if (DCI.isBeforeLegalizeOps())
3466 return SDValue();
3467
3468 if (Opc == HexagonISD::P2D) {
3469 SDValue P = Op.getOperand(0);
3470 switch (P.getOpcode()) {
3471 case HexagonISD::PTRUE:
3472 return DCI.DAG.getConstant(-1, dl, ty(Op));
3473 case HexagonISD::PFALSE:
3474 return getZero(dl, ty(Op), DCI.DAG);
3475 default:
3476 break;
3477 }
3478 } else if (Opc == ISD::VSELECT) {
3479 // This is pretty much duplicated in HexagonISelLoweringHVX...
3480 //
3481 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3482 SDValue Cond = Op.getOperand(0);
3483 if (Cond->getOpcode() == ISD::XOR) {
3484 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
3485 if (C1->getOpcode() == HexagonISD::PTRUE) {
3486 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
3487 Op.getOperand(2), Op.getOperand(1));
3488 return VSel;
3489 }
3490 }
3491 } else if (Opc == ISD::TRUNCATE) {
3492 SDValue Op0 = Op.getOperand(0);
3493 // fold (truncate (build pair x, y)) -> (truncate x) or x
3494 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3495 MVT TruncTy = ty(Op);
3496 SDValue Elem0 = Op0.getOperand(0);
3497 // if we match the low element of the pair, just return it.
3498 if (ty(Elem0) == TruncTy)
3499 return Elem0;
3500 // otherwise, if the low part is still too large, apply the truncate.
3501 if (ty(Elem0).bitsGT(TruncTy))
3502 return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);
3503 }
3504 } else if (Opc == ISD::OR) {
3505 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3506 // if s >= 32
3507 auto fold0 = [&, this](SDValue Op) {
3508 if (ty(Op) != MVT::i64)
3509 return SDValue();
3510 SDValue Shl = Op.getOperand(0);
3511 SDValue Zxt = Op.getOperand(1);
3512 if (Shl.getOpcode() != ISD::SHL)
3513 std::swap(Shl, Zxt);
3514
3515 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3516 return SDValue();
3517
3518 SDValue Z = Zxt.getOperand(0);
3519 auto *Amt = dyn_cast<ConstantSDNode>(Shl.getOperand(1));
3520 if (Amt && Amt->getZExtValue() >= 32 && ty(Z).getSizeInBits() <= 32) {
3521 unsigned A = Amt->getZExtValue();
3522 SDValue S = Shl.getOperand(0);
3523 SDValue T0 = DCI.DAG.getNode(ISD::SHL, dl, ty(S), S,
3524 DCI.DAG.getConstant(32 - A, dl, MVT::i32));
3525 SDValue T1 = DCI.DAG.getZExtOrTrunc(T0, dl, MVT::i32);
3526 SDValue T2 = DCI.DAG.getZExtOrTrunc(Z, dl, MVT::i32);
3527 return DCI.DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {T1, T2});
3528 }
3529 return SDValue();
3530 };
3531
3532 if (SDValue R = fold0(Op))
3533 return R;
3534 }
3535
3536 return SDValue();
3537}
3538
3539/// Returns relocation base for the given PIC jumptable.
3540SDValue
3542 SelectionDAG &DAG) const {
3543 int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3544 EVT VT = Table.getValueType();
3546 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T);
3547}
3548
3549//===----------------------------------------------------------------------===//
3550// Inline Assembly Support
3551//===----------------------------------------------------------------------===//
3552
3555 if (Constraint.size() == 1) {
3556 switch (Constraint[0]) {
3557 case 'q':
3558 case 'v':
3559 if (Subtarget.useHVXOps())
3560 return C_RegisterClass;
3561 break;
3562 case 'a':
3563 return C_RegisterClass;
3564 default:
3565 break;
3566 }
3567 }
3568 return TargetLowering::getConstraintType(Constraint);
3569}
3570
3571std::pair<unsigned, const TargetRegisterClass*>
3573 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3574
3575 if (Constraint.size() == 1) {
3576 switch (Constraint[0]) {
3577 case 'r': // R0-R31
3578 switch (VT.SimpleTy) {
3579 default:
3580 return {0u, nullptr};
3581 case MVT::i1:
3582 case MVT::i8:
3583 case MVT::i16:
3584 case MVT::i32:
3585 case MVT::f32:
3586 return {0u, &Hexagon::IntRegsRegClass};
3587 case MVT::i64:
3588 case MVT::f64:
3589 return {0u, &Hexagon::DoubleRegsRegClass};
3590 }
3591 break;
3592 case 'a': // M0-M1
3593 if (VT != MVT::i32)
3594 return {0u, nullptr};
3595 return {0u, &Hexagon::ModRegsRegClass};
3596 case 'q': // q0-q3
3597 switch (VT.getSizeInBits()) {
3598 default:
3599 return {0u, nullptr};
3600 case 64:
3601 case 128:
3602 return {0u, &Hexagon::HvxQRRegClass};
3603 }
3604 break;
3605 case 'v': // V0-V31
3606 switch (VT.getSizeInBits()) {
3607 default:
3608 return {0u, nullptr};
3609 case 512:
3610 return {0u, &Hexagon::HvxVRRegClass};
3611 case 1024:
3612 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3613 return {0u, &Hexagon::HvxVRRegClass};
3614 return {0u, &Hexagon::HvxWRRegClass};
3615 case 2048:
3616 return {0u, &Hexagon::HvxWRRegClass};
3617 }
3618 break;
3619 default:
3620 return {0u, nullptr};
3621 }
3622 }
3623
3624 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3625}
3626
3627/// isFPImmLegal - Returns true if the target can instruction select the
3628/// specified FP immediate natively. If false, the legalizer will
3629/// materialize the FP immediate as a load from a constant pool.
3631 bool ForCodeSize) const {
3632 return true;
3633}
3634
3635/// isLegalAddressingMode - Return true if the addressing mode represented by
3636/// AM is legal for this target, for a load/store of the specified type.
3638 const AddrMode &AM, Type *Ty,
3639 unsigned AS, Instruction *I) const {
3640 if (Ty->isSized()) {
3641 // When LSR detects uses of the same base address to access different
3642 // types (e.g. unions), it will assume a conservative type for these
3643 // uses:
3644 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3645 // The type Ty passed here would then be "void". Skip the alignment
3646 // checks, but do not return false right away, since that confuses
3647 // LSR into crashing.
3648 Align A = DL.getABITypeAlign(Ty);
3649 // The base offset must be a multiple of the alignment.
3650 if (!isAligned(A, AM.BaseOffs))
3651 return false;
3652 // The shifted offset must fit in 11 bits.
3653 if (!isInt<11>(AM.BaseOffs >> Log2(A)))
3654 return false;
3655 }
3656
3657 // No global is ever allowed as a base.
3658 if (AM.BaseGV)
3659 return false;
3660
3661 int Scale = AM.Scale;
3662 if (Scale < 0)
3663 Scale = -Scale;
3664 switch (Scale) {
3665 case 0: // No scale reg, "r+i", "r", or just "i".
3666 break;
3667 default: // No scaled addressing mode.
3668 return false;
3669 }
3670 return true;
3671}
3672
3673/// Return true if folding a constant offset with the given GlobalAddress is
3674/// legal. It is frequently not legal in PIC relocation models.
3676 const {
3677 return HTM.getRelocationModel() == Reloc::Static;
3678}
3679
3680/// isLegalICmpImmediate - Return true if the specified immediate is legal
3681/// icmp immediate, that is the target has icmp instructions which can compare
3682/// a register against the immediate without having to materialize the
3683/// immediate into a register.
3685 return Imm >= -512 && Imm <= 511;
3686}
3687
3688/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3689/// for tail call optimization. Targets which want to do tail call
3690/// optimization should implement this function.
3693 CallingConv::ID CalleeCC,
3694 bool IsVarArg,
3695 bool IsCalleeStructRet,
3696 bool IsCallerStructRet,
3698 const SmallVectorImpl<SDValue> &OutVals,
3700 SelectionDAG& DAG) const {
3701 const Function &CallerF = DAG.getMachineFunction().getFunction();
3702 CallingConv::ID CallerCC = CallerF.getCallingConv();
3703 bool CCMatch = CallerCC == CalleeCC;
3704
3705 // ***************************************************************************
3706 // Look for obvious safe cases to perform tail call optimization that do not
3707 // require ABI changes.
3708 // ***************************************************************************
3709
3710 // If this is a tail call via a function pointer, then don't do it!
3711 if (!isa<GlobalAddressSDNode>(Callee) &&
3712 !isa<ExternalSymbolSDNode>(Callee)) {
3713 return false;
3714 }
3715
3716 // Do not optimize if the calling conventions do not match and the conventions
3717 // used are not C or Fast.
3718 if (!CCMatch) {
3719 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3720 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3721 // If R & E, then ok.
3722 if (!R || !E)
3723 return false;
3724 }
3725
3726 // Do not tail call optimize vararg calls.
3727 if (IsVarArg)
3728 return false;
3729
3730 // Also avoid tail call optimization if either caller or callee uses struct
3731 // return semantics.
3732 if (IsCalleeStructRet || IsCallerStructRet)
3733 return false;
3734
3735 // In addition to the cases above, we also disable Tail Call Optimization if
3736 // the calling convention code that at least one outgoing argument needs to
3737 // go on the stack. We cannot check that here because at this point that
3738 // information is not available.
3739 return true;
3740}
3741
3742/// Returns the target specific optimal type for load and store operations as
3743/// a result of memset, memcpy, and memmove lowering.
3744///
3745/// If DstAlign is zero that means it's safe to destination alignment can
3746/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3747/// a need to check it against alignment requirement, probably because the
3748/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3749/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3750/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3751/// does not need to be loaded. It returns EVT::Other if the type should be
3752/// determined using generic target-independent logic.
3754 const MemOp &Op, const AttributeList &FuncAttributes) const {
3755 if (Op.size() >= 8 && Op.isAligned(Align(8)))
3756 return MVT::i64;
3757 if (Op.size() >= 4 && Op.isAligned(Align(4)))
3758 return MVT::i32;
3759 if (Op.size() >= 2 && Op.isAligned(Align(2)))
3760 return MVT::i16;
3761 return MVT::Other;
3762}
3763
3765 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3766 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
3767 MVT SVT = VT.getSimpleVT();
3768 if (Subtarget.isHVXVectorType(SVT, true))
3769 return allowsHvxMemoryAccess(SVT, Flags, Fast);
3771 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3772}
3773
3775 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3776 unsigned *Fast) const {
3777 MVT SVT = VT.getSimpleVT();
3778 if (Subtarget.isHVXVectorType(SVT, true))
3779 return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3780 if (Fast)
3781 *Fast = 0;
3782 return false;
3783}
3784
3785std::pair<const TargetRegisterClass*, uint8_t>
3786HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3787 MVT VT) const {
3788 if (Subtarget.isHVXVectorType(VT, true)) {
3789 unsigned BitWidth = VT.getSizeInBits();
3790 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3791
3792 if (VT.getVectorElementType() == MVT::i1)
3793 return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3794 if (BitWidth == VecWidth)
3795 return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3796 assert(BitWidth == 2 * VecWidth);
3797 return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3798 }
3799
3801}
3802
3804 ISD::LoadExtType ExtTy, EVT NewVT) const {
3805 // TODO: This may be worth removing. Check regression tests for diffs.
3806 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
3807 return false;
3808
3809 auto *L = cast<LoadSDNode>(Load);
3810 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3811 // Small-data object, do not shrink.
3812 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3813 return false;
3814 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) {
3815 auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine());
3816 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3817 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM);
3818 }
3819 return true;
3820}
3821
3823 SDNode *Node) const {
3824 AdjustHvxInstrPostInstrSelection(MI, Node);
3825}
3826
3828 Type *ValueTy, Value *Addr,
3829 AtomicOrdering Ord) const {
3830 BasicBlock *BB = Builder.GetInsertBlock();
3831 Module *M = BB->getParent()->getParent();
3832 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
3833 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
3834 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3835 : Intrinsic::hexagon_L4_loadd_locked;
3836 Function *Fn = Intrinsic::getDeclaration(M, IntID);
3837
3838 auto PtrTy = cast<PointerType>(Addr->getType());
3839 PointerType *NewPtrTy =
3840 Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace());
3841 Addr = Builder.CreateBitCast(Addr, NewPtrTy);
3842
3843 Value *Call = Builder.CreateCall(Fn, Addr, "larx");
3844
3845 return Builder.CreateBitCast(Call, ValueTy);
3846}
3847
3848/// Perform a store-conditional operation to Addr. Return the status of the
3849/// store. This should be 0 if the store succeeded, non-zero otherwise.
3851 Value *Val, Value *Addr,
3852 AtomicOrdering Ord) const {
3853 BasicBlock *BB = Builder.GetInsertBlock();
3854 Module *M = BB->getParent()->getParent();
3855 Type *Ty = Val->getType();
3856 unsigned SZ = Ty->getPrimitiveSizeInBits();
3857
3858 Type *CastTy = Builder.getIntNTy(SZ);
3859 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
3860 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3861 : Intrinsic::hexagon_S4_stored_locked;
3862 Function *Fn = Intrinsic::getDeclaration(M, IntID);
3863
3864 unsigned AS = Addr->getType()->getPointerAddressSpace();
3865 Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS));
3866 Val = Builder.CreateBitCast(Val, CastTy);
3867
3868 Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
3869 Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
3870 Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
3871 return Ext;
3872}
3873
3876 // Do not expand loads and stores that don't exceed 64 bits.
3877 return LI->getType()->getPrimitiveSizeInBits() > 64
3880}
3881
3884 // Do not expand loads and stores that don't exceed 64 bits.
3885 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
3888}
3889
3892 AtomicCmpXchgInst *AI) const {
3894}
unsigned const MachineRegisterInfo * MRI
unsigned RegSize
aarch64 promote const
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
assume Assume Builder
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define NL
uint64_t Addr
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static cl::opt< int > MaxStoresPerMemcpyCL("max-store-memcpy", cl::Hidden, cl::init(6), cl::desc("Max #stores to inline memcpy"))
static Value * getUnderLyingObjectForBrevLdIntr(Value *V)
static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo, ISD::ArgFlagsTy &ArgFlags, CCState &State)
static cl::opt< bool > AlignLoads("hexagon-align-loads", cl::Hidden, cl::init(false), cl::desc("Rewrite unaligned loads as a pair of aligned loads"))
static bool isBrevLdIntrinsic(const Value *Inst)
static cl::opt< int > MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", cl::Hidden, cl::init(4), cl::desc("Max #stores to inline memmove"))
static cl::opt< int > MaxStoresPerMemmoveCL("max-store-memmove", cl::Hidden, cl::init(6), cl::desc("Max #stores to inline memmove"))
static Value * getBrevLdObject(Value *V)
static cl::opt< int > MaxStoresPerMemsetCL("max-store-memset", cl::Hidden, cl::init(8), cl::desc("Max #stores to inline memset"))
static cl::opt< bool > DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden, cl::init(false), cl::desc("Disable minimum alignment of 1 for " "arguments passed by value on stack"))
static Value * returnEdge(const PHINode *PN, Value *IntrBaseVal)
static cl::opt< int > MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", cl::Hidden, cl::init(4), cl::desc("Max #stores to inline memcpy"))
static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl)
CreateCopyOfByValArgument - Make a copy of an aggregate at address specified by "Src" to address "Dst...
static cl::opt< int > MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", cl::Hidden, cl::init(4), cl::desc("Max #stores to inline memset"))
static cl::opt< bool > EmitJumpTables("hexagon-emit-jump-tables", cl::init(true), cl::Hidden, cl::desc("Control jump table emission on Hexagon target"))
static cl::opt< int > MinimumJumpTables("minimum-jump-tables", cl::Hidden, cl::init(5), cl::desc("Set minimum jump tables"))
static cl::opt< bool > EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden, cl::desc("Enable Hexagon SDNode scheduling"))
static cl::opt< bool > EnableFastMath("ffast-math", cl::Hidden, cl::desc("Enable Fast Math processing"))
#define Hexagon_PointerSize
#define HEXAGON_LRFP_SIZE
#define HEXAGON_GOT_SYM_NAME
IRTranslator LLVM IR MI
#define RegName(no)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
#define H(x, y, z)
Definition: MD5.cpp:57
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
#define T1
Module.h This file contains the declarations for the Module class.
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const char LLVMTargetMachineRef TM
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
@ Flags
Definition: TextStubV5.cpp:93
static llvm::Type * getVectorElementType(llvm::Type *Ty)
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1184
Class for arbitrary precision integers.
Definition: APInt.h:75
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1516
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Definition: ArrayRef.h:226
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:202
iterator end() const
Definition: ArrayRef.h:152
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
iterator begin() const
Definition: ArrayRef.h:151
const T * data() const
Definition: ArrayRef.h:160
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
The address of a basic block.
Definition: Constants.h:879
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
CCValAssign - Represent assignment of one arg/retval to a location.
bool isRegLoc() const