SIISelLowering.cpp

Go to the documentation of this file.

//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Custom DAG lowering for SI
//
//===----------------------------------------------------------------------===//
 
#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>
 
using namespace llvm;
 
#define DEBUG_TYPE "si-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
 
static cl::opt<bool> DisableLoopAlignment(
  "amdgpu-disable-loop-alignment",
  cl::desc("Do not align and prefetch loops"),
  cl::init(false));
 
static cl::opt<bool> UseDivergentRegisterIndexing(
  "amdgpu-use-divergent-register-indexing",
  cl::Hidden,
  cl::desc("Use indirect register addressing for divergent indexes"),
  cl::init(false));
 
static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
}
 
static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
}
 
static unsigned findFirstFreeSGPR(CCState &CCInfo) {
  unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
  for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
    if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
      return AMDGPU::SGPR0 + Reg;
    }
  }
  llvm_unreachable("Cannot allocate sgpr");
}
 
SITargetLowering::SITargetLowering(const TargetMachine &TM,
                                   const GCNSubtarget &STI)
    : AMDGPUTargetLowering(TM, STI),
      Subtarget(&STI) {
  addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
  addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
 
  addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
  addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass);
 
  addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
 
  const SIRegisterInfo *TRI = STI.getRegisterInfo();
  const TargetRegisterClass *V64RegClass = TRI->getVGPR64Class();
 
  addRegisterClass(MVT::f64, V64RegClass);
  addRegisterClass(MVT::v2f32, V64RegClass);
  addRegisterClass(MVT::Untyped, V64RegClass);
 
  addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
  addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96));
 
  addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass);
  addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass);
 
  addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass);
  addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128));
 
  addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
  addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160));
 
  addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass);
  addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192));
 
  addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass);
  addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192));
 
  addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass);
  addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224));
 
  addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass);
  addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256));
 
  addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass);
  addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256));
 
  addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass);
  addRegisterClass(MVT::v9f32, TRI->getVGPRClassForBitWidth(288));
 
  addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass);
  addRegisterClass(MVT::v10f32, TRI->getVGPRClassForBitWidth(320));
 
  addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass);
  addRegisterClass(MVT::v11f32, TRI->getVGPRClassForBitWidth(352));
 
  addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass);
  addRegisterClass(MVT::v12f32, TRI->getVGPRClassForBitWidth(384));
 
  addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass);
  addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512));
 
  addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass);
  addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512));
 
  addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass);
  addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024));
 
  if (Subtarget->has16BitInsts()) {
    if (Subtarget->useRealTrue16Insts()) {
      addRegisterClass(MVT::i16, &AMDGPU::VGPR_16RegClass);
      addRegisterClass(MVT::f16, &AMDGPU::VGPR_16RegClass);
      addRegisterClass(MVT::bf16, &AMDGPU::VGPR_16RegClass);
    } else {
      addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass);
      addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass);
      addRegisterClass(MVT::bf16, &AMDGPU::SReg_32RegClass);
    }
 
    // Unless there are also VOP3P operations, not operations are really legal.
    addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v2bf16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v4bf16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v8bf16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v16bf16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v32i16, &AMDGPU::SGPR_512RegClass);
    addRegisterClass(MVT::v32f16, &AMDGPU::SGPR_512RegClass);
    addRegisterClass(MVT::v32bf16, &AMDGPU::SGPR_512RegClass);
  }
 
  addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
  addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024));
 
  computeRegisterProperties(Subtarget->getRegisterInfo());
 
  // The boolean content concept here is too inflexible. Compares only ever
  // really produce a 1-bit result. Any copy/extend from these will turn into a
  // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
  // it's what most targets use.
  setBooleanContents(ZeroOrOneBooleanContent);
  setBooleanVectorContents(ZeroOrOneBooleanContent);
 
  // We need to custom lower vector stores from local memory
  setOperationAction(ISD::LOAD,
                     {MVT::v2i32,  MVT::v3i32,  MVT::v4i32,  MVT::v5i32,
                      MVT::v6i32,  MVT::v7i32,  MVT::v8i32,  MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
                      MVT::i1,     MVT::v32i32},
                     Custom);
 
  setOperationAction(ISD::STORE,
                     {MVT::v2i32,  MVT::v3i32,  MVT::v4i32,  MVT::v5i32,
                      MVT::v6i32,  MVT::v7i32,  MVT::v8i32,  MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
                      MVT::i1,     MVT::v32i32},
                     Custom);
 
  if (isTypeLegal(MVT::bf16)) {
    for (unsigned Opc :
         {ISD::FADD,     ISD::FSUB,       ISD::FMUL,    ISD::FDIV,
          ISD::FREM,     ISD::FMA,        ISD::FMINNUM, ISD::FMAXNUM,
          ISD::FMINIMUM, ISD::FMAXIMUM,   ISD::FSQRT,   ISD::FCBRT,
          ISD::FSIN,     ISD::FCOS,       ISD::FPOW,    ISD::FPOWI,
          ISD::FLDEXP,   ISD::FFREXP,     ISD::FLOG,    ISD::FLOG2,
          ISD::FLOG10,   ISD::FEXP,       ISD::FEXP2,   ISD::FEXP10,
          ISD::FCEIL,    ISD::FTRUNC,     ISD::FRINT,   ISD::FNEARBYINT,
          ISD::FROUND,   ISD::FROUNDEVEN, ISD::FFLOOR,  ISD::FCANONICALIZE,
          ISD::SETCC}) {
      // FIXME: The promoted to type shouldn't need to be explicit
      setOperationAction(Opc, MVT::bf16, Promote);
      AddPromotedToType(Opc, MVT::bf16, MVT::f32);
    }
 
    setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
 
    setOperationAction(ISD::SELECT, MVT::bf16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::bf16, MVT::i16);
 
    setOperationAction(ISD::FABS, MVT::bf16, Legal);
    setOperationAction(ISD::FNEG, MVT::bf16, Legal);
    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Legal);
 
    // We only need to custom lower because we can't specify an action for bf16
    // sources.
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
  }
 
  setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
  setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand);
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
  setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
  setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
  setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand);
  setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
  setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand);
  setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand);
  setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand);
 
  setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
  setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
  setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand);
  setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
 
  setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
 
  setOperationAction(ISD::SELECT, MVT::i1, Promote);
  setOperationAction(ISD::SELECT, MVT::i64, Custom);
  setOperationAction(ISD::SELECT, MVT::f64, Promote);
  AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
 
  setOperationAction(ISD::FSQRT, {MVT::f32, MVT::f64}, Custom);
 
  setOperationAction(ISD::SELECT_CC,
                     {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand);
 
  setOperationAction(ISD::SETCC, MVT::i1, Promote);
  setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand);
  AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
 
  setOperationAction(ISD::TRUNCATE,
                     {MVT::v2i32,  MVT::v3i32,  MVT::v4i32,  MVT::v5i32,
                      MVT::v6i32,  MVT::v7i32,  MVT::v8i32,  MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
                     Expand);
  setOperationAction(ISD::FP_ROUND,
                     {MVT::v2f32,  MVT::v3f32,  MVT::v4f32,  MVT::v5f32,
                      MVT::v6f32,  MVT::v7f32,  MVT::v8f32,  MVT::v9f32,
                      MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
                     Expand);
 
  setOperationAction(ISD::SIGN_EXTEND_INREG,
                     {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
                      MVT::v3i16, MVT::v4i16, MVT::Other},
                     Custom);
 
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::BR_CC,
                     {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand);
 
  setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Legal);
 
  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i32, Legal);
 
  setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64,
                     Expand);
 
#if 0
  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal);
#endif
 
  // We only support LOAD/STORE and vector manipulation ops for vectors
  // with > 4 elements.
  for (MVT VT :
       {MVT::v8i32,   MVT::v8f32,  MVT::v9i32,  MVT::v9f32,  MVT::v10i32,
        MVT::v10f32,  MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
        MVT::v16i32,  MVT::v16f32, MVT::v2i64,  MVT::v2f64,  MVT::v4i16,
        MVT::v4f16,   MVT::v4bf16, MVT::v3i64,  MVT::v3f64,  MVT::v6i32,
        MVT::v6f32,   MVT::v4i64,  MVT::v4f64,  MVT::v8i64,  MVT::v8f64,
        MVT::v8i16,   MVT::v8f16,  MVT::v8bf16, MVT::v16i16, MVT::v16f16,
        MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
        MVT::v32i16,  MVT::v32f16, MVT::v32bf16}) {
    for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
      switch (Op) {
      case ISD::LOAD:
      case ISD::STORE:
      case ISD::BUILD_VECTOR:
      case ISD::BITCAST:
      case ISD::UNDEF:
      case ISD::EXTRACT_VECTOR_ELT:
      case ISD::INSERT_VECTOR_ELT:
      case ISD::SCALAR_TO_VECTOR:
      case ISD::IS_FPCLASS:
        break;
      case ISD::EXTRACT_SUBVECTOR:
      case ISD::INSERT_SUBVECTOR:
      case ISD::CONCAT_VECTORS:
        setOperationAction(Op, VT, Custom);
        break;
      default:
        setOperationAction(Op, VT, Expand);
        break;
      }
    }
  }
 
  setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand);
 
  // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
  // is expanded to avoid having two separate loops in case the index is a VGPR.
 
  // Most operations are naturally 32-bit vector operations. We only support
  // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
  for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32);
  }
 
  for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32);
  }
 
  for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32);
  }
 
  for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32);
  }
 
  for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32);
  }
 
  setOperationAction(ISD::VECTOR_SHUFFLE,
                     {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32},
                     Expand);
 
  setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
                     Custom);
 
  // Avoid stack access for these.
  // TODO: Generalize to more vector types.
  setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
                     {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
                      MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
                     Custom);
 
  // Deal with vec3 vector operations when widened to vec4.
  setOperationAction(ISD::INSERT_SUBVECTOR,
                     {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom);
 
  // Deal with vec5/6/7 vector operations when widened to vec8.
  setOperationAction(ISD::INSERT_SUBVECTOR,
                     {MVT::v5i32,  MVT::v5f32,  MVT::v6i32,  MVT::v6f32,
                      MVT::v7i32,  MVT::v7f32,  MVT::v8i32,  MVT::v8f32,
                      MVT::v9i32,  MVT::v9f32,  MVT::v10i32, MVT::v10f32,
                      MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
                     Custom);
 
  // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
  // and output demarshalling
  setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom);
 
  // We can't return success/failure, only the old value,
  // let LLVM add the comparison
  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, {MVT::i32, MVT::i64},
                     Expand);
 
  setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom);
 
  setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal);
 
  // FIXME: This should be narrowed to i32, but that only happens if i64 is
  // illegal.
  // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
  setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal);
 
  // On SI this is s_memtime and s_memrealtime on VI.
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
 
  if (Subtarget->hasSMemRealTime() ||
      Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
    setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Legal);
  setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction({ISD::FPOW, ISD::FPOWI}, MVT::f16, Promote);
    setOperationAction({ISD::FLOG, ISD::FEXP, ISD::FLOG10}, MVT::f16, Custom);
  } else {
    setOperationAction(ISD::FSQRT, MVT::f16, Custom);
  }
 
  if (Subtarget->hasMadMacF32Insts())
    setOperationAction(ISD::FMAD, MVT::f32, Legal);
 
  if (!Subtarget->hasBFI())
    // fcopysign can be done in a single instruction with BFI.
    setOperationAction(ISD::FCOPYSIGN, {MVT::f32, MVT::f64}, Expand);
 
  if (!Subtarget->hasBCNT(32))
    setOperationAction(ISD::CTPOP, MVT::i32, Expand);
 
  if (!Subtarget->hasBCNT(64))
    setOperationAction(ISD::CTPOP, MVT::i64, Expand);
 
  if (Subtarget->hasFFBH())
    setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom);
 
  if (Subtarget->hasFFBL())
    setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);
 
  // We only really have 32-bit BFE instructions (and 16-bit on VI).
  //
  // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
  // effort to match them now. We want this to be false for i64 cases when the
  // extraction isn't restricted to the upper or lower half. Ideally we would
  // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
  // span the midpoint are probably relatively rare, so don't worry about them
  // for now.
  if (Subtarget->hasBFE())
    setHasExtractBitsInsn(true);
 
  // Clamp modifier on add/sub
  if (Subtarget->hasIntClamp())
    setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Legal);
 
  if (Subtarget->hasAddNoCarry())
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32},
                       Legal);
 
  setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, {MVT::f32, MVT::f64},
                     Custom);
 
  // These are really only legal for ieee_mode functions. We should be avoiding
  // them for functions that don't have ieee_mode enabled, so just say they are
  // legal.
  setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
                     {MVT::f32, MVT::f64}, Legal);
 
  if (Subtarget->haveRoundOpsF64())
    setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, MVT::f64,
                       Legal);
  else
    setOperationAction({ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
                       MVT::f64, Custom);
 
  setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
  setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, {MVT::f32, MVT::f64},
                     Legal);
  setOperationAction(ISD::FFREXP, {MVT::f32, MVT::f64}, Custom);
 
  setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FDIV}, MVT::f32, Custom);
  setOperationAction(ISD::FDIV, MVT::f64, Custom);
 
  setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand);
  setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand);
 
  // Custom lower these because we can't specify a rule based on an illegal
  // source bf16.
  setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f32, Custom);
  setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f64, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction({ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
                        ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i16, Legal);
 
    AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32);
 
    setOperationAction({ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
                       MVT::i16, Expand);
 
    setOperationAction({ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
                        ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
                        ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
                        ISD::CTPOP},
                       MVT::i16, Promote);
 
    setOperationAction(ISD::LOAD, MVT::i16, Custom);
 
    setTruncStoreAction(MVT::i64, MVT::i16, Expand);
 
    setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote);
    AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32);
    setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote);
    AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32);
 
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom);
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
 
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i32, Custom);
 
    // F16 - Constant Actions.
    setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
    setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);
 
    // F16 - Load/Store Actions.
    setOperationAction(ISD::LOAD, MVT::f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16);
    setOperationAction(ISD::STORE, MVT::f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16);
 
    // BF16 - Load/Store Actions.
    setOperationAction(ISD::LOAD, MVT::bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::bf16, MVT::i16);
    setOperationAction(ISD::STORE, MVT::bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16);
 
    // F16 - VOP1 Actions.
    setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
                        ISD::FSIN, ISD::FROUND, ISD::FPTRUNC_ROUND},
                       MVT::f16, Custom);
 
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::f16, Promote);
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::bf16, Promote);
 
    // F16 - VOP2 Actions.
    setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, {MVT::f16, MVT::bf16},
                       Expand);
    setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, MVT::f16, Custom);
    setOperationAction(ISD::FFREXP, MVT::f16, Custom);
    setOperationAction(ISD::FDIV, MVT::f16, Custom);
 
    // F16 - VOP3 Actions.
    setOperationAction(ISD::FMA, MVT::f16, Legal);
    if (STI.hasMadF16())
      setOperationAction(ISD::FMAD, MVT::f16, Legal);
 
    for (MVT VT :
         {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
          MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
          MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
        switch (Op) {
        case ISD::LOAD:
        case ISD::STORE:
        case ISD::BUILD_VECTOR:
        case ISD::BITCAST:
        case ISD::UNDEF:
        case ISD::EXTRACT_VECTOR_ELT:
        case ISD::INSERT_VECTOR_ELT:
        case ISD::INSERT_SUBVECTOR:
        case ISD::SCALAR_TO_VECTOR:
        case ISD::IS_FPCLASS:
          break;
        case ISD::EXTRACT_SUBVECTOR:
        case ISD::CONCAT_VECTORS:
          setOperationAction(Op, VT, Custom);
          break;
        default:
          setOperationAction(Op, VT, Expand);
          break;
        }
      }
    }
 
    // v_perm_b32 can handle either of these.
    setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal);
    setOperationAction(ISD::BSWAP, MVT::v4i16, Custom);
 
    // XXX - Do these do anything? Vector constants turn into build_vector.
    setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
 
    setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
                       Legal);
 
    setOperationAction(ISD::STORE, MVT::v2i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::STORE, MVT::v2f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::LOAD, MVT::v2i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::AND, MVT::v2i16, Promote);
    AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::OR, MVT::v2i16, Promote);
    AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::XOR, MVT::v2i16, Promote);
    AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32);
 
    setOperationAction(ISD::LOAD, MVT::v4i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::LOAD, MVT::v4f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32);
    setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
 
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4bf16, MVT::v2i32);
 
    setOperationAction(ISD::LOAD, MVT::v8i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32);
    setOperationAction(ISD::LOAD, MVT::v8f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32);
    setOperationAction(ISD::LOAD, MVT::v8bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8bf16, MVT::v4i32);
 
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
 
    setOperationAction(ISD::STORE, MVT::v8i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32);
    setOperationAction(ISD::STORE, MVT::v8f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32);
    setOperationAction(ISD::STORE, MVT::v8bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8bf16, MVT::v4i32);
 
    setOperationAction(ISD::LOAD, MVT::v16i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32);
    setOperationAction(ISD::LOAD, MVT::v16f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32);
    setOperationAction(ISD::LOAD, MVT::v16bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16bf16, MVT::v8i32);
 
    setOperationAction(ISD::STORE, MVT::v16i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32);
    setOperationAction(ISD::STORE, MVT::v16f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32);
    setOperationAction(ISD::STORE, MVT::v16bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16bf16, MVT::v8i32);
 
    setOperationAction(ISD::LOAD, MVT::v32i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v32i16, MVT::v16i32);
    setOperationAction(ISD::LOAD, MVT::v32f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v32f16, MVT::v16i32);
    setOperationAction(ISD::LOAD, MVT::v32bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v32bf16, MVT::v16i32);
 
    setOperationAction(ISD::STORE, MVT::v32i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v32i16, MVT::v16i32);
    setOperationAction(ISD::STORE, MVT::v32f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v32f16, MVT::v16i32);
    setOperationAction(ISD::STORE, MVT::v32bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v32bf16, MVT::v16i32);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v2i32, Expand);
    setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v4i32, Expand);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v8i32, Expand);
 
    setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
                       Subtarget->hasVOP3PInsts() ? Legal : Custom);
 
    setOperationAction(ISD::FNEG, MVT::v2f16, Legal);
    // This isn't really legal, but this avoids the legalizer unrolling it (and
    // allows matching fneg (fabs x) patterns)
    setOperationAction(ISD::FABS, MVT::v2f16, Legal);
 
    setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, MVT::f16, Custom);
    setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal);
 
    setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
                       Custom);
 
    setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
                       Expand);
 
    for (MVT Vec16 :
         {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
          MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
      setOperationAction(
          {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
          Vec16, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, Vec16, Expand);
    }
  }
 
  if (Subtarget->hasVOP3PInsts()) {
    setOperationAction({ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
                        ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
                        ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
                       MVT::v2i16, Legal);
 
    setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMINNUM_IEEE,
                        ISD::FMAXNUM_IEEE, ISD::FCANONICALIZE},
                       MVT::v2f16, Legal);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
                       Custom);
 
    setOperationAction(ISD::VECTOR_SHUFFLE,
                       {MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16,
                        MVT::v16f16, MVT::v16i16, MVT::v32f16, MVT::v32i16},
                       Custom);
 
    for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
      // Split vector operations.
      setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
                          ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
                          ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
                          ISD::SSUBSAT},
                         VT, Custom);
 
    for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
      // Split vector operations.
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE},
                         VT, Custom);
 
    setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, {MVT::v2f16, MVT::v4f16},
                       Custom);
 
    setOperationAction(ISD::FEXP, MVT::v2f16, Custom);
    setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
                       Custom);
 
    if (Subtarget->hasPackedFP32Ops()) {
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
                         MVT::v2f32, Legal);
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA},
                         {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
                         Custom);
    }
  }
 
  setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v4f16, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction(ISD::SELECT, MVT::v2i16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
  } else {
    // Legalization hack.
    setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
 
    setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v2f16, Custom);
  }
 
  setOperationAction(ISD::SELECT,
                     {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
                      MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
                      MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
                      MVT::v32f16, MVT::v32bf16},
                     Custom);
 
  setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom);
 
  if (Subtarget->hasScalarSMulU64())
    setOperationAction(ISD::MUL, MVT::i64, Custom);
 
  if (Subtarget->hasMad64_32())
    setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom);
 
  if (Subtarget->hasPrefetch())
    setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
  if (Subtarget->hasIEEEMinMax()) {
    setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM},
                       {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal);
    setOperationAction({ISD::FMINIMUM, ISD::FMAXIMUM},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
                       Custom);
  }
 
  setOperationAction(ISD::INTRINSIC_WO_CHAIN,
                     {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
                      MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
                      MVT::i8},
                     Custom);
 
  setOperationAction(ISD::INTRINSIC_W_CHAIN,
                     {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
                      MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
                      MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
                      MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
                     Custom);
 
  setOperationAction(ISD::INTRINSIC_VOID,
                     {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
                      MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
                      MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
                      MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
                     Custom);
 
  setOperationAction(ISD::STACKSAVE, MVT::Other, Custom);
  setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
  setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
  setOperationAction(ISD::GET_FPENV, MVT::i64, Custom);
  setOperationAction(ISD::SET_FPENV, MVT::i64, Custom);
 
  // TODO: Could move this to custom lowering, could benefit from combines on
  // extract of relevant bits.
  setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal);
 
  setOperationAction(ISD::MUL, MVT::i1, Promote);
 
  setTargetDAGCombine({ISD::ADD,
                       ISD::UADDO_CARRY,
                       ISD::SUB,
                       ISD::USUBO_CARRY,
                       ISD::FADD,
                       ISD::FSUB,
                       ISD::FDIV,
                       ISD::FMINNUM,
                       ISD::FMAXNUM,
                       ISD::FMINNUM_IEEE,
                       ISD::FMAXNUM_IEEE,
                       ISD::FMINIMUM,
                       ISD::FMAXIMUM,
                       ISD::FMA,
                       ISD::SMIN,
                       ISD::SMAX,
                       ISD::UMIN,
                       ISD::UMAX,
                       ISD::SETCC,
                       ISD::AND,
                       ISD::OR,
                       ISD::XOR,
                       ISD::FSHR,
                       ISD::SINT_TO_FP,
                       ISD::UINT_TO_FP,
                       ISD::FCANONICALIZE,
                       ISD::SCALAR_TO_VECTOR,
                       ISD::ZERO_EXTEND,
                       ISD::SIGN_EXTEND_INREG,
                       ISD::EXTRACT_VECTOR_ELT,
                       ISD::INSERT_VECTOR_ELT,
                       ISD::FCOPYSIGN});
 
  if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
    setTargetDAGCombine(ISD::FP_ROUND);
 
  // All memory operations. Some folding on the pointer operand is done to help
  // matching the constant offsets in the addressing modes.
  setTargetDAGCombine({ISD::LOAD,
                       ISD::STORE,
                       ISD::ATOMIC_LOAD,
                       ISD::ATOMIC_STORE,
                       ISD::ATOMIC_CMP_SWAP,
                       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
                       ISD::ATOMIC_SWAP,
                       ISD::ATOMIC_LOAD_ADD,
                       ISD::ATOMIC_LOAD_SUB,
                       ISD::ATOMIC_LOAD_AND,
                       ISD::ATOMIC_LOAD_OR,
                       ISD::ATOMIC_LOAD_XOR,
                       ISD::ATOMIC_LOAD_NAND,
                       ISD::ATOMIC_LOAD_MIN,
                       ISD::ATOMIC_LOAD_MAX,
                       ISD::ATOMIC_LOAD_UMIN,
                       ISD::ATOMIC_LOAD_UMAX,
                       ISD::ATOMIC_LOAD_FADD,
                       ISD::ATOMIC_LOAD_FMIN,
                       ISD::ATOMIC_LOAD_FMAX,
                       ISD::ATOMIC_LOAD_UINC_WRAP,
                       ISD::ATOMIC_LOAD_UDEC_WRAP,
                       ISD::INTRINSIC_VOID,
                       ISD::INTRINSIC_W_CHAIN});
 
  // FIXME: In other contexts we pretend this is a per-function property.
  setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
 
  setSchedulingPreference(Sched::RegPressure);
}
 
const GCNSubtarget *SITargetLowering::getSubtarget() const {
  return Subtarget;
}
 
ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
  static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
  return RCRegs;
}
 
//===----------------------------------------------------------------------===//
// TargetLowering queries
//===----------------------------------------------------------------------===//
 
// v_mad_mix* support a conversion from f16 to f32.
//
// There is only one special case when denormals are enabled we don't currently,
// where this is OK to use.
bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
                                       EVT DestVT, EVT SrcVT) const {
  return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
          (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
         DestVT.getScalarType() == MVT::f32 &&
         SrcVT.getScalarType() == MVT::f16 &&
         // TODO: This probably only requires no input flushing?
         denormalModeIsFlushAllF32(DAG.getMachineFunction());
}
 
bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
                                       LLT DestTy, LLT SrcTy) const {
  return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
          (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
         DestTy.getScalarSizeInBits() == 32 &&
         SrcTy.getScalarSizeInBits() == 16 &&
         // TODO: This probably only requires no input flushing?
         denormalModeIsFlushAllF32(*MI.getMF());
}
 
bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
  // SI has some legal vector types, but no legal vector operations. Say no
  // shuffles are legal in order to prefer scalarizing some vector operations.
  return false;
}
 
MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                    CallingConv::ID CC,
                                                    EVT VT) const {
  if (CC == CallingConv::AMDGPU_KERNEL)
    return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
 
  if (VT.isVector()) {
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
    if (Size == 16) {
      if (Subtarget->has16BitInsts()) {
        if (VT.isInteger())
          return MVT::v2i16;
        return (ScalarVT == MVT::bf16 ? MVT::i32 : MVT::v2f16);
      }
      return VT.isInteger() ? MVT::i32 : MVT::f32;
    }
 
    if (Size < 16)
      return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
    return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
  }
 
  if (VT.getSizeInBits() > 32)
    return MVT::i32;
 
  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
 
unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                         CallingConv::ID CC,
                                                         EVT VT) const {
  if (CC == CallingConv::AMDGPU_KERNEL)
    return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
 
  if (VT.isVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
 
    // FIXME: Should probably promote 8-bit vectors to i16.
    if (Size == 16 && Subtarget->has16BitInsts())
      return (NumElts + 1) / 2;
 
    if (Size <= 32)
      return NumElts;
 
    if (Size > 32)
      return NumElts * ((Size + 31) / 32);
  } else if (VT.getSizeInBits() > 32)
    return (VT.getSizeInBits() + 31) / 32;
 
  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
 
unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
  LLVMContext &Context, CallingConv::ID CC,
  EVT VT, EVT &IntermediateVT,
  unsigned &NumIntermediates, MVT &RegisterVT) const {
  if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
    // FIXME: We should fix the ABI to be the same on targets without 16-bit
    // support, but unless we can properly handle 3-vectors, it will be still be
    // inconsistent.
    if (Size == 16 && Subtarget->has16BitInsts()) {
      if (ScalarVT == MVT::bf16) {
        RegisterVT = MVT::i32;
        IntermediateVT = MVT::v2bf16;
      } else {
        RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
        IntermediateVT = RegisterVT;
      }
      NumIntermediates = (NumElts + 1) / 2;
      return NumIntermediates;
    }
 
    if (Size == 32) {
      RegisterVT = ScalarVT.getSimpleVT();
      IntermediateVT = RegisterVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size < 16 && Subtarget->has16BitInsts()) {
      // FIXME: Should probably form v2i16 pieces
      RegisterVT = MVT::i16;
      IntermediateVT = ScalarVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
 
    if (Size != 16 && Size <= 32) {
      RegisterVT = MVT::i32;
      IntermediateVT = ScalarVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size > 32) {
      RegisterVT = MVT::i32;
      IntermediateVT = RegisterVT;
      NumIntermediates = NumElts * ((Size + 31) / 32);
      return NumIntermediates;
    }
  }
 
  return TargetLowering::getVectorTypeBreakdownForCallingConv(
    Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
}
 
static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
                                 const DataLayout &DL, Type *Ty,
                                 unsigned MaxNumLanes) {
  assert(MaxNumLanes != 0);
 
  LLVMContext &Ctx = Ty->getContext();
  if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
    unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
    return EVT::getVectorVT(Ctx, TLI.getValueType(DL, VT->getElementType()),
                            NumElts);
  }
 
  return TLI.getValueType(DL, Ty);
}
 
// Peek through TFE struct returns to only use the data size.
static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
                                   const DataLayout &DL, Type *Ty,
                                   unsigned MaxNumLanes) {
  auto *ST = dyn_cast<StructType>(Ty);
  if (!ST)
    return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
 
  // TFE intrinsics return an aggregate type.
  assert(ST->getNumContainedTypes() == 2 &&
         ST->getContainedType(1)->isIntegerTy(32));
  return memVTFromLoadIntrData(TLI, DL, ST->getContainedType(0), MaxNumLanes);
}
 
/// Map address space 7 to MVT::v5i32 because that's its in-memory
/// representation. This return value is vector-typed because there is no
/// MVT::i160 and it is not clear if one can be added. While this could
/// cause issues during codegen, these address space 7 pointers will be
/// rewritten away by then. Therefore, we can return MVT::v5i32 in order
/// to allow pre-codegen passes that query TargetTransformInfo, often for cost
/// modeling, to work.
MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
  if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
    return MVT::v5i32;
  if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
      DL.getPointerSizeInBits(AS) == 192)
    return MVT::v6i32;
  return AMDGPUTargetLowering::getPointerTy(DL, AS);
}
/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
/// v8i32 when padding is added.
/// The in-memory representation of a p9 is {p8, i32, i32}, which is
/// also v8i32 with padding.
MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
  if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
       DL.getPointerSizeInBits(AS) == 160) ||
      (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
       DL.getPointerSizeInBits(AS) == 192))
    return MVT::v8i32;
  return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
}
 
bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                          const CallInst &CI,
                                          MachineFunction &MF,
                                          unsigned IntrID) const {
  Info.flags = MachineMemOperand::MONone;
  if (CI.hasMetadata(LLVMContext::MD_invariant_load))
    Info.flags |= MachineMemOperand::MOInvariant;
 
  if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
          AMDGPU::lookupRsrcIntrinsic(IntrID)) {
    AttributeList Attr = Intrinsic::getAttributes(CI.getContext(),
                                                  (Intrinsic::ID)IntrID);
    MemoryEffects ME = Attr.getMemoryEffects();
    if (ME.doesNotAccessMemory())
      return false;
 
    // TODO: Should images get their own address space?
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
 
    const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
    if (RsrcIntr->IsImage) {
      const AMDGPU::ImageDimIntrinsicInfo *Intr =
          AMDGPU::getImageDimIntrinsicInfo(IntrID);
      BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
      Info.align.reset();
    }
 
    Value *RsrcArg = CI.getArgOperand(RsrcIntr->RsrcArg);
    if (auto *RsrcPtrTy = dyn_cast<PointerType>(RsrcArg->getType())) {
      if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
        // We conservatively set the memory operand of a buffer intrinsic to the
        // base resource pointer, so that we can access alias information about
        // those pointers. Cases like "this points at the same value
        // but with a different offset" are handled in
        // areMemAccessesTriviallyDisjoint.
        Info.ptrVal = RsrcArg;
    }
 
    auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
    if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
      Info.flags |= MachineMemOperand::MOVolatile;
    Info.flags |= MachineMemOperand::MODereferenceable;
    if (ME.onlyReadsMemory()) {
      if (RsrcIntr->IsImage) {
        unsigned MaxNumLanes = 4;
 
        if (!BaseOpcode->Gather4) {
          // If this isn't a gather, we may have excess loaded elements in the
          // IR type. Check the dmask for the real number of elements loaded.
          unsigned DMask
            = cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
          MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
        }
 
        Info.memVT = memVTFromLoadIntrReturn(*this, MF.getDataLayout(),
                                             CI.getType(), MaxNumLanes);
      } else {
        Info.memVT =
            memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
                                    std::numeric_limits<unsigned>::max());
      }
 
      // FIXME: What does alignment mean for an image?
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.flags |= MachineMemOperand::MOLoad;
    } else if (ME.onlyWritesMemory()) {
      Info.opc = ISD::INTRINSIC_VOID;
 
      Type *DataTy = CI.getArgOperand(0)->getType();
      if (RsrcIntr->IsImage) {
        unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
        unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
        Info.memVT = memVTFromLoadIntrData(*this, MF.getDataLayout(), DataTy,
                                           DMaskLanes);
      } else
        Info.memVT = getValueType(MF.getDataLayout(), DataTy);
 
      Info.flags |= MachineMemOperand::MOStore;
    } else {
      // Atomic or NoReturn Sampler
      Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID :
                                            ISD::INTRINSIC_W_CHAIN;
      Info.flags |= MachineMemOperand::MOLoad |
                    MachineMemOperand::MOStore |
                    MachineMemOperand::MODereferenceable;
 
      switch (IntrID) {
      default:
        if (RsrcIntr->IsImage && BaseOpcode->NoReturn) {
          // Fake memory access type for no return sampler intrinsics
          Info.memVT = MVT::i32;
        } else {
          // XXX - Should this be volatile without known ordering?
          Info.flags |= MachineMemOperand::MOVolatile;
          Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
        }
        break;
      case Intrinsic::amdgcn_raw_buffer_load_lds:
      case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
      case Intrinsic::amdgcn_struct_buffer_load_lds:
      case Intrinsic::amdgcn_struct_ptr_buffer_load_lds: {
        unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
        Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
        Info.ptrVal = CI.getArgOperand(1);
        return true;
      }
      case Intrinsic::amdgcn_raw_atomic_buffer_load:
      case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
      case Intrinsic::amdgcn_struct_atomic_buffer_load:
      case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
        Info.memVT =
            memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
                                    std::numeric_limits<unsigned>::max());
        Info.flags &= ~MachineMemOperand::MOStore;
        return true;
      }
      }
    }
    return true;
  }
 
  switch (IntrID) {
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
    if (!Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    return true;
  }
  case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
  case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
    Info.ptrVal = nullptr;
    Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    return true;
  }
  case Intrinsic::amdgcn_ds_append:
  case Intrinsic::amdgcn_ds_consume: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
    if (!Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    return true;
  }
  case Intrinsic::amdgcn_global_atomic_csub: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad |
                  MachineMemOperand::MOStore |
                  MachineMemOperand::MOVolatile;
    return true;
  }
  case Intrinsic::amdgcn_image_bvh_intersect_ray: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT?
 
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad |
                  MachineMemOperand::MODereferenceable;
    return true;
  }
  case Intrinsic::amdgcn_global_atomic_fadd:
  case Intrinsic::amdgcn_global_atomic_fmin:
  case Intrinsic::amdgcn_global_atomic_fmax:
  case Intrinsic::amdgcn_global_atomic_fmin_num:
  case Intrinsic::amdgcn_global_atomic_fmax_num:
  case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
  case Intrinsic::amdgcn_flat_atomic_fadd:
  case Intrinsic::amdgcn_flat_atomic_fmin:
  case Intrinsic::amdgcn_flat_atomic_fmax:
  case Intrinsic::amdgcn_flat_atomic_fmin_num:
  case Intrinsic::amdgcn_flat_atomic_fmax_num:
  case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
  case Intrinsic::amdgcn_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad |
                  MachineMemOperand::MOStore |
                  MachineMemOperand::MODereferenceable |
                  MachineMemOperand::MOVolatile;
    return true;
  }
  case Intrinsic::amdgcn_global_load_tr_b64:
  case Intrinsic::amdgcn_global_load_tr_b128: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags |= MachineMemOperand::MOLoad;
    return true;
  }
  case Intrinsic::amdgcn_ds_gws_init:
  case Intrinsic::amdgcn_ds_gws_barrier:
  case Intrinsic::amdgcn_ds_gws_sema_v:
  case Intrinsic::amdgcn_ds_gws_sema_br:
  case Intrinsic::amdgcn_ds_gws_sema_p:
  case Intrinsic::amdgcn_ds_gws_sema_release_all: {
    Info.opc = ISD::INTRINSIC_VOID;
 
    const GCNTargetMachine &TM =
        static_cast<const GCNTargetMachine &>(getTargetMachine());
 
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    Info.ptrVal = MFI->getGWSPSV(TM);
 
    // This is an abstract access, but we need to specify a type and size.
    Info.memVT = MVT::i32;
    Info.size = 4;
    Info.align = Align(4);
 
    if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
      Info.flags |= MachineMemOperand::MOLoad;
    else
      Info.flags |= MachineMemOperand::MOStore;
    return true;
  }
  case Intrinsic::amdgcn_global_load_lds: {
    Info.opc = ISD::INTRINSIC_VOID;
    unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
    Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
    Info.ptrVal = CI.getArgOperand(1);
    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    return true;
  }
  case Intrinsic::amdgcn_ds_bvh_stack_rtn: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
 
    const GCNTargetMachine &TM =
        static_cast<const GCNTargetMachine &>(getTargetMachine());
 
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    Info.ptrVal = MFI->getGWSPSV(TM);
 
    // This is an abstract access, but we need to specify a type and size.
    Info.memVT = MVT::i32;
    Info.size = 4;
    Info.align = Align(4);
 
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    return true;
  }
  default:
    return false;
  }
}
 
void SITargetLowering::CollectTargetIntrinsicOperands(
    const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
  switch (cast<IntrinsicInst>(I).getIntrinsicID()) {
  case Intrinsic::amdgcn_addrspacecast_nonnull: {
    // The DAG's ValueType loses the addrspaces.
    // Add them as 2 extra Constant operands "from" and "to".
    unsigned SrcAS = I.getOperand(0)->getType()->getPointerAddressSpace();
    unsigned DstAS = I.getType()->getPointerAddressSpace();
    Ops.push_back(DAG.getTargetConstant(SrcAS, SDLoc(), MVT::i32));
    Ops.push_back(DAG.getTargetConstant(DstAS, SDLoc(), MVT::i32));
    break;
  }
  default:
    break;
  }
}
 
bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
                                            SmallVectorImpl<Value*> &Ops,
                                            Type *&AccessTy) const {
  Value *Ptr = nullptr;
  switch (II->getIntrinsicID()) {
  case Intrinsic::amdgcn_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_ds_append:
  case Intrinsic::amdgcn_ds_consume:
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap:
  case Intrinsic::amdgcn_flat_atomic_fadd:
  case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16:
  case Intrinsic::amdgcn_flat_atomic_fmax:
  case Intrinsic::amdgcn_flat_atomic_fmax_num:
  case Intrinsic::amdgcn_flat_atomic_fmin:
  case Intrinsic::amdgcn_flat_atomic_fmin_num:
  case Intrinsic::amdgcn_global_atomic_csub:
  case Intrinsic::amdgcn_global_atomic_fadd:
  case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
  case Intrinsic::amdgcn_global_atomic_fmax:
  case Intrinsic::amdgcn_global_atomic_fmax_num:
  case Intrinsic::amdgcn_global_atomic_fmin:
  case Intrinsic::amdgcn_global_atomic_fmin_num:
  case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
  case Intrinsic::amdgcn_global_load_tr_b64:
  case Intrinsic::amdgcn_global_load_tr_b128:
    Ptr = II->getArgOperand(0);
    break;
  case Intrinsic::amdgcn_global_load_lds:
    Ptr = II->getArgOperand(1);
    break;
  default:
    return false;
  }
  AccessTy = II->getType();
  Ops.push_back(Ptr);
  return true;
}
 
bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
                                                 unsigned AddrSpace) const {
  if (!Subtarget->hasFlatInstOffsets()) {
    // Flat instructions do not have offsets, and only have the register
    // address.
    return AM.BaseOffs == 0 && AM.Scale == 0;
  }
 
  decltype(SIInstrFlags::FLAT) FlatVariant =
      AddrSpace == AMDGPUAS::GLOBAL_ADDRESS    ? SIInstrFlags::FlatGlobal
      : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? SIInstrFlags::FlatScratch
                                               : SIInstrFlags::FLAT;
 
  return AM.Scale == 0 &&
         (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
                                  AM.BaseOffs, AddrSpace, FlatVariant));
}
 
bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
  if (Subtarget->hasFlatGlobalInsts())
    return isLegalFlatAddressingMode(AM, AMDGPUAS::GLOBAL_ADDRESS);
 
  if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
    // Assume the we will use FLAT for all global memory accesses
    // on VI.
    // FIXME: This assumption is currently wrong.  On VI we still use
    // MUBUF instructions for the r + i addressing mode.  As currently
    // implemented, the MUBUF instructions only work on buffer < 4GB.
    // It may be possible to support > 4GB buffers with MUBUF instructions,
    // by setting the stride value in the resource descriptor which would
    // increase the size limit to (stride * 4GB).  However, this is risky,
    // because it has never been validated.
    return isLegalFlatAddressingMode(AM, AMDGPUAS::FLAT_ADDRESS);
  }
 
  return isLegalMUBUFAddressingMode(AM);
}
 
bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
  // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
  // additionally can do r + r + i with addr64. 32-bit has more addressing
  // mode options. Depending on the resource constant, it can also do
  // (i64 r0) + (i32 r1) * (i14 i).
  //
  // Private arrays end up using a scratch buffer most of the time, so also
  // assume those use MUBUF instructions. Scratch loads / stores are currently
  // implemented as mubuf instructions with offen bit set, so slightly
  // different than the normal addr64.
  const SIInstrInfo *TII = Subtarget->getInstrInfo();
  if (!TII->isLegalMUBUFImmOffset(AM.BaseOffs))
    return false;
 
  // FIXME: Since we can split immediate into soffset and immediate offset,
  // would it make sense to allow any immediate?
 
  switch (AM.Scale) {
  case 0: // r + i or just i, depending on HasBaseReg.
    return true;
  case 1:
    return true; // We have r + r or r + i.
  case 2:
    if (AM.HasBaseReg) {
      // Reject 2 * r + r.
      return false;
    }
 
    // Allow 2 * r as r + r
    // Or  2 * r + i is allowed as r + r + i.
    return true;
  default: // Don't allow n * r
    return false;
  }
}
 
bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                             const AddrMode &AM, Type *Ty,
                                             unsigned AS, Instruction *I) const {
  // No global is ever allowed as a base.
  if (AM.BaseGV)
    return false;
 
  if (AS == AMDGPUAS::GLOBAL_ADDRESS)
    return isLegalGlobalAddressingMode(AM);
 
  if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
      AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
      AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
      AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
    // If the offset isn't a multiple of 4, it probably isn't going to be
    // correctly aligned.
    // FIXME: Can we get the real alignment here?
    if (AM.BaseOffs % 4 != 0)
      return isLegalMUBUFAddressingMode(AM);
 
    if (!Subtarget->hasScalarSubwordLoads()) {
      // There are no SMRD extloads, so if we have to do a small type access we
      // will use a MUBUF load.
      // FIXME?: We also need to do this if unaligned, but we don't know the
      // alignment here.
      if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
        return isLegalGlobalAddressingMode(AM);
    }
 
    if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
      // SMRD instructions have an 8-bit, dword offset on SI.
      if (!isUInt<8>(AM.BaseOffs / 4))
        return false;
    } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
      // On CI+, this can also be a 32-bit literal constant offset. If it fits
      // in 8-bits, it can use a smaller encoding.
      if (!isUInt<32>(AM.BaseOffs / 4))
        return false;
    } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
      // On VI, these use the SMEM format and the offset is 20-bit in bytes.
      if (!isUInt<20>(AM.BaseOffs))
        return false;
    } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
      // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
      // for S_BUFFER_* instructions).
      if (!isInt<21>(AM.BaseOffs))
        return false;
    } else {
      // On GFX12, all offsets are signed 24-bit in bytes.
      if (!isInt<24>(AM.BaseOffs))
        return false;
    }
 
    if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
         AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
        AM.BaseOffs < 0) {
      // Scalar (non-buffer) loads can only use a negative offset if
      // soffset+offset is non-negative. Since the compiler can only prove that
      // in a few special cases, it is safer to claim that negative offsets are
      // not supported.
      return false;
    }
 
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
      return true;
 
    if (AM.Scale == 1 && AM.HasBaseReg)
      return true;
 
    return false;
  }
 
  if (AS == AMDGPUAS::PRIVATE_ADDRESS)
    return Subtarget->enableFlatScratch()
               ? isLegalFlatAddressingMode(AM, AMDGPUAS::PRIVATE_ADDRESS)
               : isLegalMUBUFAddressingMode(AM);
 
  if (AS == AMDGPUAS::LOCAL_ADDRESS ||
      (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
    // Basic, single offset DS instructions allow a 16-bit unsigned immediate
    // field.
    // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
    // an 8-bit dword offset but we don't know the alignment here.
    if (!isUInt<16>(AM.BaseOffs))
      return false;
 
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
      return true;
 
    if (AM.Scale == 1 && AM.HasBaseReg)
      return true;
 
    return false;
  }
 
  if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
    // For an unknown address space, this usually means that this is for some
    // reason being used for pure arithmetic, and not based on some addressing
    // computation. We don't have instructions that compute pointers with any
    // addressing modes, so treat them as having no offset like flat
    // instructions.
    return isLegalFlatAddressingMode(AM, AMDGPUAS::FLAT_ADDRESS);
  }
 
  // Assume a user alias of global for unknown address spaces.
  return isLegalGlobalAddressingMode(AM);
}
 
bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
                                        const MachineFunction &MF) const {
  if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
    return (MemVT.getSizeInBits() <= 4 * 32);
  if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
    unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
    return (MemVT.getSizeInBits() <= MaxPrivateBits);
  }
  if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
    return (MemVT.getSizeInBits() <= 2 * 32);
  return true;
}
 
bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
    unsigned Size, unsigned AddrSpace, Align Alignment,
    MachineMemOperand::Flags Flags, unsigned *IsFast) const {
  if (IsFast)
    *IsFast = 0;
 
  if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
      AddrSpace == AMDGPUAS::REGION_ADDRESS) {
    // Check if alignment requirements for ds_read/write instructions are
    // disabled.
    if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
      return false;
 
    Align RequiredAlignment(PowerOf2Ceil(Size/8)); // Natural alignment.
    if (Subtarget->hasLDSMisalignedBug() && Size > 32 &&
        Alignment < RequiredAlignment)
      return false;
 
    // Either, the alignment requirements are "enabled", or there is an
    // unaligned LDS access related hardware bug though alignment requirements
    // are "disabled". In either case, we need to check for proper alignment
    // requirements.
    //
    switch (Size) {
    case 64:
      // SI has a hardware bug in the LDS / GDS bounds checking: if the base
      // address is negative, then the instruction is incorrectly treated as
      // out-of-bounds even if base + offsets is in bounds. Split vectorized
      // loads here to avoid emitting ds_read2_b32. We may re-combine the
      // load later in the SILoadStoreOptimizer.
      if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
        return false;
 
      // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
      // can do a 4 byte aligned, 8 byte access in a single operation using
      // ds_read2/write2_b32 with adjacent offsets.
      RequiredAlignment = Align(4);
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
        // ds_write2_b32 depending on the alignment. In either case with either
        // alignment there is no faster way of doing this.
 
        // The numbers returned here and below are not additive, it is a 'speed
        // rank'. They are just meant to be compared to decide if a certain way
        // of lowering an operation is faster than another. For that purpose
        // naturally aligned operation gets it bitsize to indicate that "it
        // operates with a speed comparable to N-bit wide load". With the full
        // alignment ds128 is slower than ds96 for example. If underaligned it
        // is comparable to a speed of a single dword access, which would then
        // mean 32 < 128 and it is faster to issue a wide load regardless.
        // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
        // wider load which will not be aligned anymore the latter is slower.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 64
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    case 96:
      if (!Subtarget->hasDS96AndDS128())
        return false;
 
      // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
      // gfx8 and older.
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // Naturally aligned access is fastest. However, also report it is Fast
        // if memory is aligned less than DWORD. A narrow load or store will be
        // be equally slow as a single ds_read_b96/ds_write_b96, but there will
        // be more of them, so overall we will pay less penalty issuing a single
        // instruction.
 
        // See comment on the values above.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 96
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    case 128:
      if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
        return false;
 
      // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
      // gfx8 and older, but  we can do a 8 byte aligned, 16 byte access in a
      // single operation using ds_read2/write2_b64.
      RequiredAlignment = Align(8);
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // Naturally aligned access is fastest. However, also report it is Fast
        // if memory is aligned less than DWORD. A narrow load or store will be
        // be equally slow as a single ds_read_b128/ds_write_b128, but there
        // will be more of them, so overall we will pay less penalty issuing a
        // single instruction.
 
        // See comment on the values above.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 128
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    default:
      if (Size > 32)
        return false;
 
      break;
    }
 
    // See comment on the values above.
    // Note that we have a single-dword or sub-dword here, so if underaligned
    // it is a slowest possible access, hence returned value is 0.
    if (IsFast)
      *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
 
    return Alignment >= RequiredAlignment ||
           Subtarget->hasUnalignedDSAccessEnabled();
  }
 
  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
    bool AlignedBy4 = Alignment >= Align(4);
    if (IsFast)
      *IsFast = AlignedBy4;
 
    return AlignedBy4 ||
           Subtarget->enableFlatScratch() ||
           Subtarget->hasUnalignedScratchAccess();
  }
 
  // FIXME: We have to be conservative here and assume that flat operations
  // will access scratch.  If we had access to the IR function, then we
  // could determine if any private memory was used in the function.
  if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
      !Subtarget->hasUnalignedScratchAccess()) {
    bool AlignedBy4 = Alignment >= Align(4);
    if (IsFast)
      *IsFast = AlignedBy4;
 
    return AlignedBy4;
  }
 
  // So long as they are correct, wide global memory operations perform better
  // than multiple smaller memory ops -- even when misaligned
  if (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace)) {
    if (IsFast)
      *IsFast = Size;
 
    return Alignment >= Align(4) ||
           Subtarget->hasUnalignedBufferAccessEnabled();
  }
 
  // Smaller than dword value must be aligned.
  if (Size < 32)
    return false;
 
  // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
  // byte-address are ignored, thus forcing Dword alignment.
  // This applies to private, global, and constant memory.
  if (IsFast)
    *IsFast = 1;
 
  return Size >= 32 && Alignment >= Align(4);
}
 
bool SITargetLowering::allowsMisalignedMemoryAccesses(
    EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    unsigned *IsFast) const {
  return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace,
                                            Alignment, Flags, IsFast);
}
 
EVT SITargetLowering::getOptimalMemOpType(
    const MemOp &Op, const AttributeList &FuncAttributes) const {
  // FIXME: Should account for address space here.
 
  // The default fallback uses the private pointer size as a guess for a type to
  // use. Make sure we switch these to 64-bit accesses.
 
  if (Op.size() >= 16 &&
      Op.isDstAligned(Align(4))) // XXX: Should only do for global
    return MVT::v4i32;
 
  if (Op.size() >= 8 && Op.isDstAligned(Align(4)))
    return MVT::v2i32;
 
  // Use the default.
  return MVT::Other;
}
 
bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
  const MemSDNode *MemNode = cast<MemSDNode>(N);
  return MemNode->getMemOperand()->getFlags() & MONoClobber;
}
 
bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
  return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
         AS == AMDGPUAS::PRIVATE_ADDRESS;
}
 
bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
                                           unsigned DestAS) const {
  // Flat -> private/local is a simple truncate.
  // Flat -> global is no-op
  if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
    return true;
 
  const GCNTargetMachine &TM =
      static_cast<const GCNTargetMachine &>(getTargetMachine());
  return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
}
 
bool SITargetLowering::isMemOpUniform(const SDNode *N) const {
  const MemSDNode *MemNode = cast<MemSDNode>(N);
 
  return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand());
}
 
TargetLoweringBase::LegalizeTypeAction
SITargetLowering::getPreferredVectorAction(MVT VT) const {
  if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
      VT.getScalarType().bitsLE(MVT::i16))
    return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                                         Type *Ty) const {
  // FIXME: Could be smarter if called for vector constants.
  return true;
}
 
bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
                                               unsigned Index) const {
  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
    return false;
 
  // TODO: Add more cases that are cheap.
  return Index == 0;
}
 
bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
  if (Subtarget->has16BitInsts() && VT == MVT::i16) {
    switch (Op) {
    case ISD::LOAD:
    case ISD::STORE:
 
    // These operations are done with 32-bit instructions anyway.
    case ISD::AND:
    case ISD::OR:
    case ISD::XOR:
    case ISD::SELECT:
      // TODO: Extensions?
      return true;
    default:
      return false;
    }
  }
 
  // SimplifySetCC uses this function to determine whether or not it should
  // create setcc with i1 operands.  We don't have instructions for i1 setcc.
  if (VT == MVT::i1 && Op == ISD::SETCC)
    return false;
 
  return TargetLowering::isTypeDesirableForOp(Op, VT);
}
 
SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
                                                   const SDLoc &SL,
                                                   SDValue Chain,
                                                   uint64_t Offset) const {
  const DataLayout &DL = DAG.getDataLayout();
  MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  const ArgDescriptor *InputPtrReg;
  const TargetRegisterClass *RC;
  LLT ArgTy;
  MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 
  std::tie(InputPtrReg, RC, ArgTy) =
      Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
 
  // We may not have the kernarg segment argument if we have no kernel
  // arguments.
  if (!InputPtrReg)
    return DAG.getConstant(Offset, SL, PtrVT);
 
  MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
  SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
    MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
 
  return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset));
}
 
SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
                                            const SDLoc &SL) const {
  uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(),
                                               FIRST_IMPLICIT);
  return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
}
 
SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
                                         const SDLoc &SL) const {
 
  Function &F = DAG.getMachineFunction().getFunction();
  std::optional<uint32_t> KnownSize =
      AMDGPUMachineFunction::getLDSKernelIdMetadata(F);
  if (KnownSize.has_value())
    return DAG.getConstant(*KnownSize, SL, MVT::i32);
  return SDValue();
}
 
SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
                                         const SDLoc &SL, SDValue Val,
                                         bool Signed,
                                         const ISD::InputArg *Arg) const {
  // First, if it is a widened vector, narrow it.
  if (VT.isVector() &&
      VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
    EVT NarrowedVT =
        EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(),
                         VT.getVectorNumElements());
    Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
                      DAG.getConstant(0, SL, MVT::i32));
  }
 
  // Then convert the vector elements or scalar value.
  if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) &&
      VT.bitsLT(MemVT)) {
    unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
    Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
  }
 
  if (MemVT.isFloatingPoint())
    Val = getFPExtOrFPRound(DAG, Val, SL, VT);
  else if (Signed)
    Val = DAG.getSExtOrTrunc(Val, SL, VT);
  else
    Val = DAG.getZExtOrTrunc(Val, SL, VT);
 
  return Val;
}
 
SDValue SITargetLowering::lowerKernargMemParameter(
    SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
    uint64_t Offset, Align Alignment, bool Signed,
    const ISD::InputArg *Arg) const {
  MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
 
  // Try to avoid using an extload by loading earlier than the argument address,
  // and extracting the relevant bits. The load should hopefully be merged with
  // the previous argument.
  if (MemVT.getStoreSize() < 4 && Alignment < 4) {
    // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
    int64_t AlignDownOffset = alignDown(Offset, 4);
    int64_t OffsetDiff = Offset - AlignDownOffset;
 
    EVT IntVT = MemVT.changeTypeToInteger();
 
    // TODO: If we passed in the base kernel offset we could have a better
    // alignment than 4, but we don't really need it.
    SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
    SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4),
                               MachineMemOperand::MODereferenceable |
                                   MachineMemOperand::MOInvariant);
 
    SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
    SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
 
    SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
    ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
    ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
 
 
    return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
  }
 
  SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
  SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
                             MachineMemOperand::MODereferenceable |
                                 MachineMemOperand::MOInvariant);
 
  SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
  return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
}
 
SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
                                              const SDLoc &SL, SDValue Chain,
                                              const ISD::InputArg &Arg) const {
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  if (Arg.Flags.isByVal()) {
    unsigned Size = Arg.Flags.getByValSize();
    int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
    return DAG.getFrameIndex(FrameIdx, MVT::i32);
  }
 
  unsigned ArgOffset = VA.getLocMemOffset();
  unsigned ArgSize = VA.getValVT().getStoreSize();
 
  int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
 
  // Create load nodes to retrieve arguments from the stack.
  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
  SDValue ArgValue;
 
  // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
  ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
  MVT MemVT = VA.getValVT();
 
  switch (VA.getLocInfo()) {
  default:
    break;
  case CCValAssign::BCvt:
    MemVT = VA.getLocVT();
    break;
  case CCValAssign::SExt:
    ExtType = ISD::SEXTLOAD;
    break;
  case CCValAssign::ZExt:
    ExtType = ISD::ZEXTLOAD;
    break;
  case CCValAssign::AExt:
    ExtType = ISD::EXTLOAD;
    break;
  }
 
  ArgValue = DAG.getExtLoad(
    ExtType, SL, VA.getLocVT(), Chain, FIN,
    MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
    MemVT);
  return ArgValue;
}
 
SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
  const SIMachineFunctionInfo &MFI,
  EVT VT,
  AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
  const ArgDescriptor *Reg = nullptr;
  const TargetRegisterClass *RC;
  LLT Ty;
 
  CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
  const ArgDescriptor WorkGroupIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP9);
  // If GridZ is not programmed in an entry function then the hardware will set
  // it to all zeros, so there is no need to mask the GridY value in the low
  // order bits.
  const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
      AMDGPU::TTMP7,
      AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
  const ArgDescriptor WorkGroupIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
  if (Subtarget->hasArchitectedSGPRs() &&
      (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
    switch (PVID) {
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
      Reg = &WorkGroupIDX;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
      Reg = &WorkGroupIDY;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
      Reg = &WorkGroupIDZ;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    default:
      break;
    }
  }
 
  if (!Reg)
    std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID);
  if (!Reg) {
    if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
      // It's possible for a kernarg intrinsic call to appear in a kernel with
      // no allocated segment, in which case we do not add the user sgpr
      // argument, so just return null.
      return DAG.getConstant(0, SDLoc(), VT);
    }
 
    // It's undefined behavior if a function marked with the amdgpu-no-*
    // attributes uses the corresponding intrinsic.
    return DAG.getUNDEF(VT);
  }
 
  return loadInputValue(DAG, RC, VT, SDLoc(DAG.getEntryNode()), *Reg);
}
 
static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
                               CallingConv::ID CallConv,
                               ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
                               FunctionType *FType,
                               SIMachineFunctionInfo *Info) {
  for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
    const ISD::InputArg *Arg = &Ins[I];
 
    assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
           "vector type argument should have been split");
 
    // First check if it's a PS input addr.
    if (CallConv == CallingConv::AMDGPU_PS &&
        !Arg->Flags.isInReg() && PSInputNum <= 15) {
      bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum);
 
      // Inconveniently only the first part of the split is marked as isSplit,
      // so skip to the end. We only want to increment PSInputNum once for the
      // entire split argument.
      if (Arg->Flags.isSplit()) {
        while (!Arg->Flags.isSplitEnd()) {
          assert((!Arg->VT.isVector() ||
                  Arg->VT.getScalarSizeInBits() == 16) &&
                 "unexpected vector split in ps argument type");
          if (!SkipArg)
            Splits.push_back(*Arg);
          Arg = &Ins[++I];
        }
      }
 
      if (SkipArg) {
        // We can safely skip PS inputs.
        Skipped.set(Arg->getOrigArgIndex());
        ++PSInputNum;
        continue;
      }
 
      Info->markPSInputAllocated(PSInputNum);
      if (Arg->Used)
        Info->markPSInputEnabled(PSInputNum);
 
      ++PSInputNum;
    }
 
    Splits.push_back(*Arg);
  }
}
 
// Allocate special inputs passed in VGPRs.
void SITargetLowering::allocateSpecialEntryInputVGPRs(CCState &CCInfo,
                                                      MachineFunction &MF,
                                                      const SIRegisterInfo &TRI,
                                                      SIMachineFunctionInfo &Info) const {
  const LLT S32 = LLT::scalar(32);
  MachineRegisterInfo &MRI = MF.getRegInfo();
 
  if (Info.hasWorkItemIDX()) {
    Register Reg = AMDGPU::VGPR0;
    MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
    CCInfo.AllocateReg(Reg);
    unsigned Mask = (Subtarget->hasPackedTID() &&
                     Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
    Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
  }
 
  if (Info.hasWorkItemIDY()) {
    assert(Info.hasWorkItemIDX());
    if (Subtarget->hasPackedTID()) {
      Info.setWorkItemIDY(ArgDescriptor::createRegister(AMDGPU::VGPR0,
                                                        0x3ff << 10));
    } else {
      unsigned Reg = AMDGPU::VGPR1;
      MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
      CCInfo.AllocateReg(Reg);
      Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
    }
  }
 
  if (Info.hasWorkItemIDZ()) {
    assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
    if (Subtarget->hasPackedTID()) {
      Info.setWorkItemIDZ(ArgDescriptor::createRegister(AMDGPU::VGPR0,
                                                        0x3ff << 20));
    } else {
      unsigned Reg = AMDGPU::VGPR2;
      MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
      CCInfo.AllocateReg(Reg);
      Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
    }
  }
}
 
// Try to allocate a VGPR at the end of the argument list, or if no argument
// VGPRs are left allocating a stack slot.
// If \p Mask is is given it indicates bitfield position in the register.
// If \p Arg is given use it with new ]p Mask instead of allocating new.
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
                                         ArgDescriptor Arg = ArgDescriptor()) {
  if (Arg.isSet())
    return ArgDescriptor::createArg(Arg, Mask);
 
  ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
  if (RegIdx == ArgVGPRs.size()) {
    // Spill to stack required.
    int64_t Offset = CCInfo.AllocateStack(4, Align(4));
 
    return ArgDescriptor::createStack(Offset, Mask);
  }
 
  unsigned Reg = ArgVGPRs[RegIdx];
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
 
  MachineFunction &MF = CCInfo.getMachineFunction();
  Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
  MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32));
  return ArgDescriptor::createRegister(Reg, Mask);
}
 
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
                                             const TargetRegisterClass *RC,
                                             unsigned NumArgRegs) {
  ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
  if (RegIdx == ArgSGPRs.size())
    report_fatal_error("ran out of SGPRs for arguments");
 
  unsigned Reg = ArgSGPRs[RegIdx];
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
 
  MachineFunction &MF = CCInfo.getMachineFunction();
  MF.addLiveIn(Reg, RC);
  return ArgDescriptor::createRegister(Reg);
}
 
// If this has a fixed position, we still should allocate the register in the
// CCInfo state. Technically we could get away with this for values passed
// outside of the normal argument range.
static void allocateFixedSGPRInputImpl(CCState &CCInfo,
                                       const TargetRegisterClass *RC,
                                       MCRegister Reg) {
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
  MachineFunction &MF = CCInfo.getMachineFunction();
  MF.addLiveIn(Reg, RC);
}
 
static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
  if (Arg) {
    allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass,
                               Arg.getRegister());
  } else
    Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
}
 
static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
  if (Arg) {
    allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass,
                               Arg.getRegister());
  } else
    Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
}
 
/// Allocate implicit function VGPR arguments at the end of allocated user
/// arguments.
void SITargetLowering::allocateSpecialInputVGPRs(
  CCState &CCInfo, MachineFunction &MF,
  const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
  const unsigned Mask = 0x3ff;
  ArgDescriptor Arg;
 
  if (Info.hasWorkItemIDX()) {
    Arg = allocateVGPR32Input(CCInfo, Mask);
    Info.setWorkItemIDX(Arg);
  }
 
  if (Info.hasWorkItemIDY()) {
    Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
    Info.setWorkItemIDY(Arg);
  }
 
  if (Info.hasWorkItemIDZ())
    Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
}
 
/// Allocate implicit function VGPR arguments in fixed registers.
void SITargetLowering::allocateSpecialInputVGPRsFixed(
  CCState &CCInfo, MachineFunction &MF,
  const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
  Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31);
  if (!Reg)
    report_fatal_error("failed to allocated VGPR for implicit arguments");
 
  const unsigned Mask = 0x3ff;
  Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
  Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10));
  Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20));
}
 
void SITargetLowering::allocateSpecialInputSGPRs(
  CCState &CCInfo,
  MachineFunction &MF,
  const SIRegisterInfo &TRI,
  SIMachineFunctionInfo &Info) const {
  auto &ArgInfo = Info.getArgInfo();
  const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
 
  // TODO: Unify handling with private memory pointers.
  if (UserSGPRInfo.hasDispatchPtr())
    allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr);
 
  const Module *M = MF.getFunction().getParent();
  if (UserSGPRInfo.hasQueuePtr() &&
      AMDGPU::getAMDHSACodeObjectVersion(*M) < AMDGPU::AMDHSA_COV5)
    allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr);
 
  // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
  // constant offset from the kernarg segment.
  if (Info.hasImplicitArgPtr())
    allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr);
 
  if (UserSGPRInfo.hasDispatchID())
    allocateSGPR64Input(CCInfo, ArgInfo.DispatchID);
 
  // flat_scratch_init is not applicable for non-kernel functions.
 
  if (Info.hasWorkGroupIDX())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX);
 
  if (Info.hasWorkGroupIDY())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY);
 
  if (Info.hasWorkGroupIDZ())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ);
 
  if (Info.hasLDSKernelId())
    allocateSGPR32Input(CCInfo, ArgInfo.LDSKernelId);
}
 
// Allocate special inputs passed in user SGPRs.
void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
                                            MachineFunction &MF,
                                            const SIRegisterInfo &TRI,
                                            SIMachineFunctionInfo &Info) const {
  const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
  if (UserSGPRInfo.hasImplicitBufferPtr()) {
    Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
    MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(ImplicitBufferPtrReg);
  }
 
  // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
  if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
    Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
    MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
    CCInfo.AllocateReg(PrivateSegmentBufferReg);
  }
 
  if (UserSGPRInfo.hasDispatchPtr()) {
    Register DispatchPtrReg = Info.addDispatchPtr(TRI);
    MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(DispatchPtrReg);
  }
 
  const Module *M = MF.getFunction().getParent();
  if (UserSGPRInfo.hasQueuePtr() &&
      AMDGPU::getAMDHSACodeObjectVersion(*M) < AMDGPU::AMDHSA_COV5) {
    Register QueuePtrReg = Info.addQueuePtr(TRI);
    MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(QueuePtrReg);
  }
 
  if (UserSGPRInfo.hasKernargSegmentPtr()) {
    MachineRegisterInfo &MRI = MF.getRegInfo();
    Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
    CCInfo.AllocateReg(InputPtrReg);
 
    Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
    MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
  }
 
  if (UserSGPRInfo.hasDispatchID()) {
    Register DispatchIDReg = Info.addDispatchID(TRI);
    MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(DispatchIDReg);
  }
 
  if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
    Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
    MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(FlatScratchInitReg);
  }
 
  if (UserSGPRInfo.hasPrivateSegmentSize()) {
    Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
    MF.addLiveIn(PrivateSegmentSizeReg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(PrivateSegmentSizeReg);
  }
 
  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
  // these from the dispatch pointer.
}
 
// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
// sequential starting from the first argument.
void SITargetLowering::allocatePreloadKernArgSGPRs(
    CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
    const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
    const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
  Function &F = MF.getFunction();
  unsigned LastExplicitArgOffset =
      MF.getSubtarget<GCNSubtarget>().getExplicitKernelArgOffset();
  GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
  bool InPreloadSequence = true;
  unsigned InIdx = 0;
  for (auto &Arg : F.args()) {
    if (!InPreloadSequence || !Arg.hasInRegAttr())
      break;
 
    int ArgIdx = Arg.getArgNo();
    // Don't preload non-original args or parts not in the current preload
    // sequence.
    if (InIdx < Ins.size() && (!Ins[InIdx].isOrigArg() ||
                               (int)Ins[InIdx].getOrigArgIndex() != ArgIdx))
      break;
 
    for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
           (int)Ins[InIdx].getOrigArgIndex() == ArgIdx;
         InIdx++) {
      assert(ArgLocs[ArgIdx].isMemLoc());
      auto &ArgLoc = ArgLocs[InIdx];
      const Align KernelArgBaseAlign = Align(16);
      unsigned ArgOffset = ArgLoc.getLocMemOffset();
      Align Alignment = commonAlignment(KernelArgBaseAlign, ArgOffset);
      unsigned NumAllocSGPRs =
          alignTo(ArgLoc.getLocVT().getFixedSizeInBits(), 32) / 32;
 
      // Arg is preloaded into the previous SGPR.
      if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
        Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
            Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
        continue;
      }
 
      unsigned Padding = ArgOffset - LastExplicitArgOffset;
      unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
      // Check for free user SGPRs for preloading.
      if (PaddingSGPRs + NumAllocSGPRs + 1 /*Synthetic SGPRs*/ >
          SGPRInfo.getNumFreeUserSGPRs()) {
        InPreloadSequence = false;
        break;
      }
 
      // Preload this argument.
      const TargetRegisterClass *RC =
          TRI.getSGPRClassForBitWidth(NumAllocSGPRs * 32);
      SmallVectorImpl<MCRegister> *PreloadRegs =
          Info.addPreloadedKernArg(TRI, RC, NumAllocSGPRs, InIdx, PaddingSGPRs);
 
      if (PreloadRegs->size() > 1)
        RC = &AMDGPU::SGPR_32RegClass;
      for (auto &Reg : *PreloadRegs) {
        assert(Reg);
        MF.addLiveIn(Reg, RC);
        CCInfo.AllocateReg(Reg);
      }
 
      LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
    }
  }
}
 
void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
                                           const SIRegisterInfo &TRI,
                                           SIMachineFunctionInfo &Info) const {
  // Always allocate this last since it is a synthetic preload.
  if (Info.hasLDSKernelId()) {
    Register Reg = Info.addLDSKernelId();
    MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(Reg);
  }
}
 
// Allocate special input registers that are initialized per-wave.
void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo,
                                           MachineFunction &MF,
                                           SIMachineFunctionInfo &Info,
                                           CallingConv::ID CallConv,
                                           bool IsShader) const {
  bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
  if (Subtarget->hasUserSGPRInit16Bug() && !IsShader) {
    // Note: user SGPRs are handled by the front-end for graphics shaders
    // Pad up the used user SGPRs with dead inputs.
 
    // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
    // before enabling architected SGPRs for workgroup IDs.
    assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
 
    unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
    // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
    // rely on it to reach 16 since if we end up having no stack usage, it will
    // not really be added.
    unsigned NumRequiredSystemSGPRs = Info.hasWorkGroupIDX() +
                                      Info.hasWorkGroupIDY() +
                                      Info.hasWorkGroupIDZ() +
                                      Info.hasWorkGroupInfo();
    for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
      Register Reg = Info.addReservedUserSGPR();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
  }
 
  if (!HasArchitectedSGPRs) {
    if (Info.hasWorkGroupIDX()) {
      Register Reg = Info.addWorkGroupIDX();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
 
    if (Info.hasWorkGroupIDY()) {
      Register Reg = Info.addWorkGroupIDY();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
 
    if (Info.hasWorkGroupIDZ()) {
      Register Reg = Info.addWorkGroupIDZ();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
  }
 
  if (Info.hasWorkGroupInfo()) {
    Register Reg = Info.addWorkGroupInfo();
    MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(Reg);
  }
 
  if (Info.hasPrivateSegmentWaveByteOffset()) {
    // Scratch wave offset passed in system SGPR.
    unsigned PrivateSegmentWaveByteOffsetReg;
 
    if (IsShader) {
      PrivateSegmentWaveByteOffsetReg =
        Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
 
      // This is true if the scratch wave byte offset doesn't have a fixed
      // location.
      if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
        PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
        Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
      }
    } else
      PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
 
    MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
  }
 
  assert(!Subtarget->hasUserSGPRInit16Bug() || IsShader ||
         Info.getNumPreloadedSGPRs() >= 16);
}
 
static void reservePrivateMemoryRegs(const TargetMachine &TM,
                                     MachineFunction &MF,
                                     const SIRegisterInfo &TRI,
                                     SIMachineFunctionInfo &Info) {
  // Now that we've figured out where the scratch register inputs are, see if
  // should reserve the arguments and use them directly.
  MachineFrameInfo &MFI = MF.getFrameInfo();
  bool HasStackObjects = MFI.hasStackObjects();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
  // Record that we know we have non-spill stack objects so we don't need to
  // check all stack objects later.
  if (HasStackObjects)
    Info.setHasNonSpillStackObjects(true);
 
  // Everything live out of a block is spilled with fast regalloc, so it's
  // almost certain that spilling will be required.
  if (TM.getOptLevel() == CodeGenOptLevel::None)
    HasStackObjects = true;
 
  // For now assume stack access is needed in any callee functions, so we need
  // the scratch registers to pass in.
  bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
 
  if (!ST.enableFlatScratch()) {
    if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
      // If we have stack objects, we unquestionably need the private buffer
      // resource. For the Code Object V2 ABI, this will be the first 4 user
      // SGPR inputs. We can reserve those and use them directly.
 
      Register PrivateSegmentBufferReg =
          Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
      Info.setScratchRSrcReg(PrivateSegmentBufferReg);
    } else {
      unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
      // We tentatively reserve the last registers (skipping the last registers
      // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
      // we'll replace these with the ones immediately after those which were
      // really allocated. In the prologue copies will be inserted from the
      // argument to these reserved registers.
 
      // Without HSA, relocations are used for the scratch pointer and the
      // buffer resource setup is always inserted in the prologue. Scratch wave
      // offset is still in an input SGPR.
      Info.setScratchRSrcReg(ReservedBufferReg);
    }
  }
 
  MachineRegisterInfo &MRI = MF.getRegInfo();
 
  // For entry functions we have to set up the stack pointer if we use it,
  // whereas non-entry functions get this "for free". This means there is no
  // intrinsic advantage to using S32 over S34 in cases where we do not have
  // calls but do need a frame pointer (i.e. if we are requested to have one
  // because frame pointer elimination is disabled). To keep things simple we
  // only ever use S32 as the call ABI stack pointer, and so using it does not
  // imply we need a separate frame pointer.
  //
  // Try to use s32 as the SP, but move it if it would interfere with input
  // arguments. This won't work with calls though.
  //
  // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
  // registers.
  if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
    Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
  } else {
    assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
 
    if (MFI.hasCalls())
      report_fatal_error("call in graphics shader with too many input SGPRs");
 
    for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
      if (!MRI.isLiveIn(Reg)) {
        Info.setStackPtrOffsetReg(Reg);
        break;
      }
    }
 
    if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
      report_fatal_error("failed to find register for SP");
  }
 
  // hasFP should be accurate for entry functions even before the frame is
  // finalized, because it does not rely on the known stack size, only
  // properties like whether variable sized objects are present.
  if (ST.getFrameLowering()->hasFP(MF)) {
    Info.setFrameOffsetReg(AMDGPU::SGPR33);
  }
}
 
bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
  const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
  return !Info->isEntryFunction();
}
 
void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
 
}
 
void SITargetLowering::insertCopiesSplitCSR(
  MachineBasicBlock *Entry,
  const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
  if (!IStart)
    return;
 
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
  MachineBasicBlock::iterator MBBI = Entry->begin();
  for (const MCPhysReg *I = IStart; *I; ++I) {
    const TargetRegisterClass *RC = nullptr;
    if (AMDGPU::SReg_64RegClass.contains(*I))
      RC = &AMDGPU::SGPR_64RegClass;
    else if (AMDGPU::SReg_32RegClass.contains(*I))
      RC = &AMDGPU::SGPR_32RegClass;
    else
      llvm_unreachable("Unexpected register class in CSRsViaCopy!");
 
    Register NewVR = MRI->createVirtualRegister(RC);
    // Create copy from CSR to a virtual register.
    Entry->addLiveIn(*I);
    BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
      .addReg(*I);
 
    // Insert the copy-back instructions right before the terminator.
    for (auto *Exit : Exits)
      BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
              TII->get(TargetOpcode::COPY), *I)
        .addReg(NewVR);
  }
}
 
SDValue SITargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &Fn = MF.getFunction();
  FunctionType *FType = MF.getFunction().getFunctionType();
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) {
    DiagnosticInfoUnsupported NoGraphicsHSA(
        Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc());
    DAG.getContext()->diagnose(NoGraphicsHSA);
    return DAG.getEntryNode();
  }
 
  SmallVector<ISD::InputArg, 16> Splits;
  SmallVector<CCValAssign, 16> ArgLocs;
  BitVector Skipped(Ins.size());
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                 *DAG.getContext());
 
  bool IsGraphics = AMDGPU::isGraphics(CallConv);
  bool IsKernel = AMDGPU::isKernel(CallConv);
  bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
 
  if (IsGraphics) {
    const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
    assert(!UserSGPRInfo.hasDispatchPtr() &&
           !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
           !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
           !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
    (void)UserSGPRInfo;
    if (!Subtarget->enableFlatScratch())
      assert(!UserSGPRInfo.hasFlatScratchInit());
    if ((CallConv != CallingConv::AMDGPU_CS &&
         CallConv != CallingConv::AMDGPU_Gfx) ||
        !Subtarget->hasArchitectedSGPRs())
      assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
             !Info->hasWorkGroupIDZ());
  }
 
  if (CallConv == CallingConv::AMDGPU_PS) {
    processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
 
    // At least one interpolation mode must be enabled or else the GPU will
    // hang.
    //
    // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
    // set PSInputAddr, the user wants to enable some bits after the compilation
    // based on run-time states. Since we can't know what the final PSInputEna
    // will look like, so we shouldn't do anything here and the user should take
    // responsibility for the correct programming.
    //
    // Otherwise, the following restrictions apply:
    // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
    // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
    //   enabled too.
    if ((Info->getPSInputAddr() & 0x7F) == 0 ||
        ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) {
      CCInfo.AllocateReg(AMDGPU::VGPR0);
      CCInfo.AllocateReg(AMDGPU::VGPR1);
      Info->markPSInputAllocated(0);
      Info->markPSInputEnabled(0);
    }
    if (Subtarget->isAmdPalOS()) {
      // For isAmdPalOS, the user does not enable some bits after compilation
      // based on run-time states; the register values being generated here are
      // the final ones set in hardware. Therefore we need to apply the
      // workaround to PSInputAddr and PSInputEnable together.  (The case where
      // a bit is set in PSInputAddr but not PSInputEnable is where the
      // frontend set up an input arg for a particular interpolation mode, but
      // nothing uses that input arg. Really we should have an earlier pass
      // that removes such an arg.)
      unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
      if ((PsInputBits & 0x7F) == 0 ||
          ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
        Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));
    }
  } else if (IsKernel) {
    assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
  } else {
    Splits.append(Ins.begin(), Ins.end());
  }
 
  if (IsKernel)
    analyzeFormalArgumentsCompute(CCInfo, Ins);
 
  if (IsEntryFunc) {
    allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
    allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
    if (IsKernel && Subtarget->hasKernargPreload())
      allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, *TRI, *Info);
 
    allocateLDSKernelId(CCInfo, MF, *TRI, *Info);
  } else if (!IsGraphics) {
    // For the fixed ABI, pass workitem IDs in the last argument register.
    allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
 
    // FIXME: Sink this into allocateSpecialInputSGPRs
    if (!Subtarget->enableFlatScratch())
      CCInfo.AllocateReg(Info->getScratchRSrcReg());
 
    allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
  }
 
  if (!IsKernel) {
    CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
    CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
  }
 
  SmallVector<SDValue, 16> Chains;
 
  // FIXME: This is the minimum kernel argument alignment. We should improve
  // this to the maximum alignment of the arguments.
  //
  // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
  // kern arg offset.
  const Align KernelArgBaseAlign = Align(16);
 
  for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
    const ISD::InputArg &Arg = Ins[i];
    if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) {
      InVals.push_back(DAG.getUNDEF(Arg.VT));
      continue;
    }
 
    CCValAssign &VA = ArgLocs[ArgIdx++];
    MVT VT = VA.getLocVT();
 
    if (IsEntryFunc && VA.isMemLoc()) {
      VT = Ins[i].VT;
      EVT MemVT = VA.getLocVT();
 
      const uint64_t Offset = VA.getLocMemOffset();
      Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
 
      if (Arg.Flags.isByRef()) {
        SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
 
        const GCNTargetMachine &TM =
            static_cast<const GCNTargetMachine &>(getTargetMachine());
        if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
                                    Arg.Flags.getPointerAddrSpace())) {
          Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
                                     Arg.Flags.getPointerAddrSpace());
        }
 
        InVals.push_back(Ptr);
        continue;
      }
 
      SDValue NewArg;
      if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(i)) {
        if (MemVT.getStoreSize() < 4 && Alignment < 4) {
          // In this case the argument is packed into the previous preload SGPR.
          int64_t AlignDownOffset = alignDown(Offset, 4);
          int64_t OffsetDiff = Offset - AlignDownOffset;
          EVT IntVT = MemVT.changeTypeToInteger();
 
          const SIMachineFunctionInfo *Info =
              MF.getInfo<SIMachineFunctionInfo>();
          MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
          Register Reg =
              Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs[0];
 
          assert(Reg);
          Register VReg = MRI.getLiveInVirtReg(Reg);
          SDValue Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
 
          SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, DL, MVT::i32);
          SDValue Extract = DAG.getNode(ISD::SRL, DL, MVT::i32, Copy, ShiftAmt);
 
          SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, Extract);
          ArgVal = DAG.getNode(ISD::BITCAST, DL, MemVT, ArgVal);
          NewArg = convertArgType(DAG, VT, MemVT, DL, ArgVal,
                                  Ins[i].Flags.isSExt(), &Ins[i]);
 
          NewArg = DAG.getMergeValues({NewArg, Copy.getValue(1)}, DL);
        } else {
          const SIMachineFunctionInfo *Info =
              MF.getInfo<SIMachineFunctionInfo>();
          MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
          const SmallVectorImpl<MCRegister> &PreloadRegs =
              Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs;
 
          SDValue Copy;
          if (PreloadRegs.size() == 1) {
            Register VReg = MRI.getLiveInVirtReg(PreloadRegs[0]);
            const TargetRegisterClass *RC = MRI.getRegClass(VReg);
            NewArg = DAG.getCopyFromReg(
                Chain, DL, VReg,
                EVT::getIntegerVT(*DAG.getContext(),
                                  TRI->getRegSizeInBits(*RC)));
 
          } else {
            // If the kernarg alignment does not match the alignment of the SGPR
            // tuple RC that can accommodate this argument, it will be built up
            // via copies from from the individual SGPRs that the argument was
            // preloaded to.
            SmallVector<SDValue, 4> Elts;
            for (auto Reg : PreloadRegs) {
              Register VReg = MRI.getLiveInVirtReg(Reg);
              Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
              Elts.push_back(Copy);
            }
            NewArg =
                DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i32,
                                                    PreloadRegs.size()),
                                   DL, Elts);
          }
 
          // If the argument was preloaded to multiple consecutive 32-bit
          // registers because of misalignment between addressable SGPR tuples
          // and the argument size, we can still assume that because of kernarg
          // segment alignment restrictions that NewArg's size is the same as
          // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
          // truncate since we cannot preload to less than a single SGPR and the
          // MemVT may be smaller.
          EVT MemVTInt =
              EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
          if (MemVT.bitsLT(NewArg.getSimpleValueType()))
            NewArg = DAG.getNode(ISD::TRUNCATE, DL, MemVTInt, NewArg);
 
          NewArg = DAG.getBitcast(MemVT, NewArg);
          NewArg = convertArgType(DAG, VT, MemVT, DL, NewArg,
                                  Ins[i].Flags.isSExt(), &Ins[i]);
          NewArg = DAG.getMergeValues({NewArg, Chain}, DL);
        }
      } else {
        NewArg =
            lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset,
                                     Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
      }
      Chains.push_back(NewArg.getValue(1));
 
      auto *ParamTy =
        dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
      if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
          ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
                      ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
        // On SI local pointers are just offsets into LDS, so they are always
        // less than 16-bits.  On CI and newer they could potentially be
        // real pointers, so we can't guarantee their size.
        NewArg = DAG.getNode(ISD::AssertZext, DL, NewArg.getValueType(), NewArg,
                             DAG.getValueType(MVT::i16));
      }
 
      InVals.push_back(NewArg);
      continue;
    }
    if (!IsEntryFunc && VA.isMemLoc()) {
      SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
      InVals.push_back(Val);
      if (!Arg.Flags.isByVal())
        Chains.push_back(Val.getValue(1));
      continue;
    }
 
    assert(VA.isRegLoc() && "Parameter must be in a register!");
 
    Register Reg = VA.getLocReg();
    const TargetRegisterClass *RC = nullptr;
    if (AMDGPU::VGPR_32RegClass.contains(Reg))
      RC = &AMDGPU::VGPR_32RegClass;
    else if (AMDGPU::SGPR_32RegClass.contains(Reg))
      RC = &AMDGPU::SGPR_32RegClass;
    else
      llvm_unreachable("Unexpected register class in LowerFormalArguments!");
    EVT ValVT = VA.getValVT();
 
    Reg = MF.addLiveIn(Reg, RC);
    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
 
    if (Arg.Flags.isSRet()) {
      // The return object should be reasonably addressable.
 
      // FIXME: This helps when the return is a real sret. If it is a
      // automatically inserted sret (i.e. CanLowerReturn returns false), an
      // extra copy is inserted in SelectionDAGBuilder which obscures this.
      unsigned NumBits
        = 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
      Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
        DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
    }
 
    // If this is an 8 or 16-bit value, it is really passed promoted
    // to 32 bits. Insert an assert[sz]ext to capture this, then
    // truncate to the right size.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val);
      break;
    case CCValAssign::SExt:
      Val = DAG.getNode(ISD::AssertSext, DL, VT, Val,
                        DAG.getValueType(ValVT));
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
      break;
    case CCValAssign::ZExt:
      Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
                        DAG.getValueType(ValVT));
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
      break;
    case CCValAssign::AExt:
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    InVals.push_back(Val);
  }
 
  // Start adding system SGPRs.
  if (IsEntryFunc)
    allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics);
 
  // DAG.getPass() returns nullptr when using new pass manager.
  // TODO: Use DAG.getMFAM() to access analysis result.
  if (DAG.getPass()) {
    auto &ArgUsageInfo = DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
    ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo());
  }
 
  unsigned StackArgSize = CCInfo.getStackSize();
  Info->setBytesInStackArgArea(StackArgSize);
 
  return Chains.empty() ? Chain :
    DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
}
 
// TODO: If return values can't fit in registers, we should return as many as
// possible in registers before passing on stack.
bool SITargetLowering::CanLowerReturn(
  CallingConv::ID CallConv,
  MachineFunction &MF, bool IsVarArg,
  const SmallVectorImpl<ISD::OutputArg> &Outs,
  LLVMContext &Context) const {
  // Replacing returns with sret/stack usage doesn't make sense for shaders.
  // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
  // for shaders. Vector types should be explicitly handled by CC.
  if (AMDGPU::isEntryFunctionCC(CallConv))
    return true;
 
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
  if (!CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg)))
    return false;
 
  // We must use the stack if return would require unavailable registers.
  unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
  unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs();
  for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
    if (CCInfo.isAllocated(AMDGPU::VGPR_32RegClass.getRegister(i)))
      return false;
 
  return true;
}
 
SDValue
SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                              bool isVarArg,
                              const SmallVectorImpl<ISD::OutputArg> &Outs,
                              const SmallVectorImpl<SDValue> &OutVals,
                              const SDLoc &DL, SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  if (AMDGPU::isKernel(CallConv)) {
    return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
                                             OutVals, DL, DAG);
  }
 
  bool IsShader = AMDGPU::isShader(CallConv);
 
  Info->setIfReturnsVoid(Outs.empty());
  bool IsWaveEnd = Info->returnsVoid() && IsShader;
 
  // CCValAssign - represent the assignment of the return value to a location.
  SmallVector<CCValAssign, 48> RVLocs;
  SmallVector<ISD::OutputArg, 48> Splits;
 
  // CCState - Info about the registers and stack slots.
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
 
  // Analyze outgoing return values.
  CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
 
  SDValue Glue;
  SmallVector<SDValue, 48> RetOps;
  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
 
  // Copy the result values into the output registers.
  for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
       ++I, ++RealRVLocIdx) {
    CCValAssign &VA = RVLocs[I];
    assert(VA.isRegLoc() && "Can only return in registers!");
    // TODO: Partially return in registers if return values don't fit.
    SDValue Arg = OutVals[RealRVLocIdx];
 
    // Copied from other backends.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue);
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
  }
 
  // FIXME: Does sret work properly?
  if (!Info->isEntryFunction()) {
    const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
    const MCPhysReg *I =
      TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
    if (I) {
      for (; *I; ++I) {
        if (AMDGPU::SReg_64RegClass.contains(*I))
          RetOps.push_back(DAG.getRegister(*I, MVT::i64));
        else if (AMDGPU::SReg_32RegClass.contains(*I))
          RetOps.push_back(DAG.getRegister(*I, MVT::i32));
        else
          llvm_unreachable("Unexpected register class in CSRsViaCopy!");
      }
    }
  }
 
  // Update chain and glue.
  RetOps[0] = Chain;
  if (Glue.getNode())
    RetOps.push_back(Glue);
 
  unsigned Opc = AMDGPUISD::ENDPGM;
  if (!IsWaveEnd)
    Opc = IsShader ? AMDGPUISD::RETURN_TO_EPILOG : AMDGPUISD::RET_GLUE;
  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
}
 
SDValue SITargetLowering::LowerCallResult(
    SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
    SDValue ThisVal) const {
  CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
 
  // Assign locations to each value returned by this call.
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeCallResult(Ins, RetCC);
 
  // Copy all of the result registers out of their specified physreg.
  for (CCValAssign VA : RVLocs) {
    SDValue Val;
 
    if (VA.isRegLoc()) {
      Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InGlue);
      Chain = Val.getValue(1);
      InGlue = Val.getValue(2);
    } else if (VA.isMemLoc()) {
      report_fatal_error("TODO: return values in memory");
    } else
      llvm_unreachable("unknown argument location type");
 
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::ZExt:
      Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::SExt:
      Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::AExt:
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    InVals.push_back(Val);
  }
 
  return Chain;
}
 
// Add code to pass special inputs required depending on used features separate
// from the explicit user arguments present in the IR.
void SITargetLowering::passSpecialInputs(
    CallLoweringInfo &CLI,
    CCState &CCInfo,
    const SIMachineFunctionInfo &Info,
    SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
    SmallVectorImpl<SDValue> &MemOpChains,
    SDValue Chain) const {
  // If we don't have a call site, this was a call inserted by
  // legalization. These can never use special inputs.
  if (!CLI.CB)
    return;
 
  SelectionDAG &DAG = CLI.DAG;
  const SDLoc &DL = CLI.DL;
  const Function &F = DAG.getMachineFunction().getFunction();
 
  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
  const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
 
  const AMDGPUFunctionArgInfo *CalleeArgInfo
    = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
  if (const Function *CalleeFunc = CLI.CB->getCalledFunction()) {
    // DAG.getPass() returns nullptr when using new pass manager.
    // TODO: Use DAG.getMFAM() to access analysis result.
    if (DAG.getPass()) {
      auto &ArgUsageInfo =
          DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
      CalleeArgInfo = &ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc);
    }
  }
 
  // TODO: Unify with private memory register handling. This is complicated by
  // the fact that at least in kernels, the input argument is not necessarily
  // in the same location as the input.
  static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
                             StringLiteral> ImplicitAttrs[] = {
    {AMDGPUFunctionArgInfo::DISPATCH_PTR, "amdgpu-no-dispatch-ptr"},
    {AMDGPUFunctionArgInfo::QUEUE_PTR, "amdgpu-no-queue-ptr" },
    {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, "amdgpu-no-implicitarg-ptr"},
    {AMDGPUFunctionArgInfo::DISPATCH_ID, "amdgpu-no-dispatch-id"},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, "amdgpu-no-workgroup-id-x"},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,"amdgpu-no-workgroup-id-y"},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,"amdgpu-no-workgroup-id-z"},
    {AMDGPUFunctionArgInfo::LDS_KERNEL_ID,"amdgpu-no-lds-kernel-id"},
  };
 
  for (auto Attr : ImplicitAttrs) {
    const ArgDescriptor *OutgoingArg;
    const TargetRegisterClass *ArgRC;
    LLT ArgTy;
 
    AMDGPUFunctionArgInfo::PreloadedValue InputID = Attr.first;
 
    // If the callee does not use the attribute value, skip copying the value.
    if (CLI.CB->hasFnAttr(Attr.second))
      continue;
 
    std::tie(OutgoingArg, ArgRC, ArgTy) =
        CalleeArgInfo->getPreloadedValue(InputID);
    if (!OutgoingArg)
      continue;
 
    const ArgDescriptor *IncomingArg;
    const TargetRegisterClass *IncomingArgRC;
    LLT Ty;
    std::tie(IncomingArg, IncomingArgRC, Ty) =
        CallerArgInfo.getPreloadedValue(InputID);
    assert(IncomingArgRC == ArgRC);
 
    // All special arguments are ints for now.
    EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
    SDValue InputReg;
 
    if (IncomingArg) {
      InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
    } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
      // The implicit arg ptr is special because it doesn't have a corresponding
      // input for kernels, and is computed from the kernarg segment pointer.
      InputReg = getImplicitArgPtr(DAG, DL);
    } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
      std::optional<uint32_t> Id =
          AMDGPUMachineFunction::getLDSKernelIdMetadata(F);
      if (Id.has_value()) {
        InputReg = DAG.getConstant(*Id, DL, ArgVT);
      } else {
        InputReg = DAG.getUNDEF(ArgVT);
      }
    } else {
      // We may have proven the input wasn't needed, although the ABI is
      // requiring it. We just need to allocate the register appropriately.
      InputReg = DAG.getUNDEF(ArgVT);
    }
 
    if (OutgoingArg->isRegister()) {
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
      if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
        report_fatal_error("failed to allocate implicit input argument");
    } else {
      unsigned SpecialArgOffset =
          CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4));
      SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
                                              SpecialArgOffset);
      MemOpChains.push_back(ArgStore);
    }
  }
 
  // Pack workitem IDs into a single register or pass it as is if already
  // packed.
  const ArgDescriptor *OutgoingArg;
  const TargetRegisterClass *ArgRC;
  LLT Ty;
 
  std::tie(OutgoingArg, ArgRC, Ty) =
      CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
  if (!OutgoingArg)
    std::tie(OutgoingArg, ArgRC, Ty) =
        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
  if (!OutgoingArg)
    std::tie(OutgoingArg, ArgRC, Ty) =
        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
  if (!OutgoingArg)
    return;
 
  const ArgDescriptor *IncomingArgX = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
  const ArgDescriptor *IncomingArgY = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
  const ArgDescriptor *IncomingArgZ = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
 
  SDValue InputReg;
  SDLoc SL;
 
  const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-x");
  const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-y");
  const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-z");
 
  // If incoming ids are not packed we need to pack them.
  if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX &&
      NeedWorkItemIDX) {
    if (Subtarget->getMaxWorkitemID(F, 0) != 0) {
      InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
    } else {
      InputReg = DAG.getConstant(0, DL, MVT::i32);
    }
  }
 
  if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY &&
      NeedWorkItemIDY && Subtarget->getMaxWorkitemID(F, 1) != 0) {
    SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
    Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
                    DAG.getShiftAmountConstant(10, MVT::i32, SL));
    InputReg = InputReg.getNode() ?
                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y;
  }
 
  if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ &&
      NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(F, 2) != 0) {
    SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
    Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
                    DAG.getShiftAmountConstant(20, MVT::i32, SL));
    InputReg = InputReg.getNode() ?
                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z;
  }
 
  if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
    if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
      // We're in a situation where the outgoing function requires the workitem
      // ID, but the calling function does not have it (e.g a graphics function
      // calling a C calling convention function). This is illegal, but we need
      // to produce something.
      InputReg = DAG.getUNDEF(MVT::i32);
    } else {
      // Workitem ids are already packed, any of present incoming arguments
      // will carry all required fields.
      ArgDescriptor IncomingArg = ArgDescriptor::createArg(
        IncomingArgX ? *IncomingArgX :
        IncomingArgY ? *IncomingArgY :
        *IncomingArgZ, ~0u);
      InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
    }
  }
 
  if (OutgoingArg->isRegister()) {
    if (InputReg)
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
 
    CCInfo.AllocateReg(OutgoingArg->getRegister());
  } else {
    unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4));
    if (InputReg) {
      SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
                                              SpecialArgOffset);
      MemOpChains.push_back(ArgStore);
    }
  }
}
 
static bool canGuaranteeTCO(CallingConv::ID CC) {
  return CC == CallingConv::Fast;
}
 
/// Return true if we might ever do TCO for calls with this calling convention.
static bool mayTailCallThisCC(CallingConv::ID CC) {
  switch (CC) {
  case CallingConv::C:
  case CallingConv::AMDGPU_Gfx:
    return true;
  default:
    return canGuaranteeTCO(CC);
  }
}
 
bool SITargetLowering::isEligibleForTailCallOptimization(
    SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals,
    const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
  if (AMDGPU::isChainCC(CalleeCC))
    return true;
 
  if (!mayTailCallThisCC(CalleeCC))
    return false;
 
  // For a divergent call target, we need to do a waterfall loop over the
  // possible callees which precludes us from using a simple jump.
  if (Callee->isDivergent())
    return false;
 
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &CallerF = MF.getFunction();
  CallingConv::ID CallerCC = CallerF.getCallingConv();
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
 
  // Kernels aren't callable, and don't have a live in return address so it
  // doesn't make sense to do a tail call with entry functions.
  if (!CallerPreserved)
    return false;
 
  bool CCMatch = CallerCC == CalleeCC;
 
  if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
    if (canGuaranteeTCO(CalleeCC) && CCMatch)
      return true;
    return false;
  }
 
  // TODO: Can we handle var args?
  if (IsVarArg)
    return false;
 
  for (const Argument &Arg : CallerF.args()) {
    if (Arg.hasByValAttr())
      return false;
  }
 
  LLVMContext &Ctx = *DAG.getContext();
 
  // Check that the call results are passed in the same way.
  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
                                  CCAssignFnForCall(CalleeCC, IsVarArg),
                                  CCAssignFnForCall(CallerCC, IsVarArg)))
    return false;
 
  // The callee has to preserve all registers the caller needs to preserve.
  if (!CCMatch) {
    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
      return false;
  }
 
  // Nothing more to check if the callee is taking no arguments.
  if (Outs.empty())
    return true;
 
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
 
  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
 
  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
  // If the stack arguments for this call do not fit into our own save area then
  // the call cannot be made tail.
  // TODO: Is this really necessary?
  if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
    return false;
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
  return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
}
 
bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
  if (!CI->isTailCall())
    return false;
 
  const Function *ParentFn = CI->getParent()->getParent();
  if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv()))
    return false;
  return true;
}
 
// The wave scratch offset register is used as the global base pointer.
SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
                                    SmallVectorImpl<SDValue> &InVals) const {
  CallingConv::ID CallConv = CLI.CallConv;
  bool IsChainCallConv = AMDGPU::isChainCC(CallConv);
 
  SelectionDAG &DAG = CLI.DAG;
 
  TargetLowering::ArgListEntry RequestedExec;
  if (IsChainCallConv) {
    // The last argument should be the value that we need to put in EXEC.
    // Pop it out of CLI.Outs and CLI.OutVals before we do any processing so we
    // don't treat it like the rest of the arguments.
    RequestedExec = CLI.Args.back();
    assert(RequestedExec.Node && "No node for EXEC");
 
    if (!RequestedExec.Ty->isIntegerTy(Subtarget->getWavefrontSize()))
      return lowerUnhandledCall(CLI, InVals, "Invalid value for EXEC");
 
    assert(CLI.Outs.back().OrigArgIndex == 2 && "Unexpected last arg");
    CLI.Outs.pop_back();
    CLI.OutVals.pop_back();
 
    if (RequestedExec.Ty->isIntegerTy(64)) {
      assert(CLI.Outs.back().OrigArgIndex == 2 && "Exec wasn't split up");
      CLI.Outs.pop_back();
      CLI.OutVals.pop_back();
    }
 
    assert(CLI.Outs.back().OrigArgIndex != 2 &&
           "Haven't popped all the pieces of the EXEC mask");
  }
 
  const SDLoc &DL = CLI.DL;
  SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
  SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
  SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
  SDValue Chain = CLI.Chain;
  SDValue Callee = CLI.Callee;
  bool &IsTailCall = CLI.IsTailCall;
  bool IsVarArg = CLI.IsVarArg;
  bool IsSibCall = false;
  MachineFunction &MF = DAG.getMachineFunction();
 
  if (Callee.isUndef() || isNullConstant(Callee)) {
    if (!CLI.IsTailCall) {
      for (ISD::InputArg &Arg : CLI.Ins)
        InVals.push_back(DAG.getUNDEF(Arg.VT));
    }
 
    return Chain;
  }
 
  if (IsVarArg) {
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported call to variadic function ");
  }
 
  if (!CLI.CB)
    report_fatal_error("unsupported libcall legalization");
 
  if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported required tail call to function ");
  }
 
  if (IsTailCall) {
    IsTailCall = isEligibleForTailCallOptimization(
      Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
    if (!IsTailCall &&
        ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
      report_fatal_error("failed to perform tail call elimination on a call "
                         "site marked musttail or on llvm.amdgcn.cs.chain");
    }
 
    bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
 
    // A sibling call is one where we're under the usual C ABI and not planning
    // to change that but can still do a tail call:
    if (!TailCallOpt && IsTailCall)
      IsSibCall = true;
 
    if (IsTailCall)
      ++NumTailCalls;
  }
 
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
  SmallVector<SDValue, 8> MemOpChains;
 
  // Analyze operands of the call, assigning locations to each operand.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
 
  if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CallConv)) {
    // With a fixed ABI, allocate fixed registers before user arguments.
    passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
  }
 
  CCInfo.AnalyzeCallOperands(Outs, AssignFn);
 
  // Get a count of how many bytes are to be pushed on the stack.
  unsigned NumBytes = CCInfo.getStackSize();
 
  if (IsSibCall) {
    // Since we're not changing the ABI to make this a tail call, the memory
    // operands are already available in the caller's incoming argument space.
    NumBytes = 0;
  }
 
  // FPDiff is the byte offset of the call's argument area from the callee's.
  // Stores to callee stack arguments will be placed in FixedStackSlots offset
  // by this amount for a tail call. In a sibling call it must be 0 because the
  // caller will deallocate the entire stack and the callee still expects its
  // arguments to begin at SP+0. Completely unused for non-tail calls.
  int32_t FPDiff = 0;
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  // Adjust the stack pointer for the new arguments...
  // These operations are automatically eliminated by the prolog/epilog pass
  if (!IsSibCall)
    Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
 
  if (!IsSibCall || IsChainCallConv) {
    if (!Subtarget->enableFlatScratch()) {
      SmallVector<SDValue, 4> CopyFromChains;
 
      // In the HSA case, this should be an identity copy.
      SDValue ScratchRSrcReg
        = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
      RegsToPass.emplace_back(IsChainCallConv
                                  ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
                                  : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
                              ScratchRSrcReg);
      CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
      Chain = DAG.getTokenFactor(DL, CopyFromChains);
    }
  }
 
  MVT PtrVT = MVT::i32;
 
  // Walk the register/memloc assignments, inserting copies/loads.
  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
    CCValAssign &VA = ArgLocs[i];
    SDValue Arg = OutVals[i];
 
    // Promote the value if needed.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::FPExt:
      Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    if (VA.isRegLoc()) {
      RegsToPass.push_back(std::pair(VA.getLocReg(), Arg));
    } else {
      assert(VA.isMemLoc());
 
      SDValue DstAddr;
      MachinePointerInfo DstInfo;
 
      unsigned LocMemOffset = VA.getLocMemOffset();
      int32_t Offset = LocMemOffset;
 
      SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
      MaybeAlign Alignment;
 
      if (IsTailCall) {
        ISD::ArgFlagsTy Flags = Outs[i].Flags;
        unsigned OpSize = Flags.isByVal() ?
          Flags.getByValSize() : VA.getValVT().getStoreSize();
 
        // FIXME: We can have better than the minimum byval required alignment.
        Alignment =
            Flags.isByVal()
                ? Flags.getNonZeroByValAlign()
                : commonAlignment(Subtarget->getStackAlignment(), Offset);
 
        Offset = Offset + FPDiff;
        int FI = MFI.CreateFixedObject(OpSize, Offset, true);
 
        DstAddr = DAG.getFrameIndex(FI, PtrVT);
        DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
 
        // Make sure any stack arguments overlapping with where we're storing
        // are loaded before this eventual operation. Otherwise they'll be
        // clobbered.
 
        // FIXME: Why is this really necessary? This seems to just result in a
        // lot of code to copy the stack and write them back to the same
        // locations, which are supposed to be immutable?
        Chain = addTokenForArgument(Chain, DAG, MFI, FI);
      } else {
        // Stores to the argument stack area are relative to the stack pointer.
        SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(),
                                        MVT::i32);
        DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff);
        DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
        Alignment =
            commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
      }
 
      if (Outs[i].Flags.isByVal()) {
        SDValue SizeNode =
            DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
        SDValue Cpy =
            DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
                          Outs[i].Flags.getNonZeroByValAlign(),
                          /*isVol = */ false, /*AlwaysInline = */ true,
                          /*CI=*/nullptr, std::nullopt, DstInfo,
                          MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
 
        MemOpChains.push_back(Cpy);
      } else {
        SDValue Store =
            DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment);
        MemOpChains.push_back(Store);
      }
    }
  }
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
  for (auto &RegToPass : RegsToPass) {
    Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
                             RegToPass.second, InGlue);
    InGlue = Chain.getValue(1);
  }
 
 
  // We don't usually want to end the call-sequence here because we would tidy
  // the frame up *after* the call, however in the ABI-changing tail-call case
  // we've carefully laid out the parameters so that when sp is reset they'll be
  // in the correct location.
  if (IsTailCall && !IsSibCall) {
    Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InGlue, DL);
    InGlue = Chain.getValue(1);
  }
 
  std::vector<SDValue> Ops;
  Ops.push_back(Chain);
  Ops.push_back(Callee);
  // Add a redundant copy of the callee global which will not be legalized, as
  // we need direct access to the callee later.
  if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) {
    const GlobalValue *GV = GSD->getGlobal();
    Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
  } else {
    Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
  }
 
  if (IsTailCall) {
    // Each tail call may have to adjust the stack by a different amount, so
    // this information must travel along with the operation for eventual
    // consumption by emitEpilogue.
    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
  }
 
  if (IsChainCallConv)
    Ops.push_back(RequestedExec.Node);
 
  // Add argument registers to the end of the list so that they are known live
  // into the call.
  for (auto &RegToPass : RegsToPass) {
    Ops.push_back(DAG.getRegister(RegToPass.first,
                                  RegToPass.second.getValueType()));
  }
 
  // Add a register mask operand representing the call-preserved registers.
  auto *TRI = static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
  assert(Mask && "Missing call preserved mask for calling convention");
  Ops.push_back(DAG.getRegisterMask(Mask));
 
  if (SDValue Token = CLI.ConvergenceControlToken) {
    SmallVector<SDValue, 2> GlueOps;
    GlueOps.push_back(Token);
    if (InGlue)
      GlueOps.push_back(InGlue);
 
    InGlue = SDValue(DAG.getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, DL,
                                        MVT::Glue, GlueOps),
                     0);
  }
 
  if (InGlue)
    Ops.push_back(InGlue);
 
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
  // If we're doing a tall call, use a TC_RETURN here rather than an
  // actual call instruction.
  if (IsTailCall) {
    MFI.setHasTailCall();
    unsigned OPC = AMDGPUISD::TC_RETURN;
    switch (CallConv) {
    case CallingConv::AMDGPU_Gfx:
      OPC = AMDGPUISD::TC_RETURN_GFX;
      break;
    case CallingConv::AMDGPU_CS_Chain:
    case CallingConv::AMDGPU_CS_ChainPreserve:
      OPC = AMDGPUISD::TC_RETURN_CHAIN;
      break;
    }
 
    return DAG.getNode(OPC, DL, NodeTys, Ops);
  }
 
  // Returns a chain and a flag for retval copy to use.
  SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
  Chain = Call.getValue(0);
  InGlue = Call.getValue(1);
 
  uint64_t CalleePopBytes = NumBytes;
  Chain = DAG.getCALLSEQ_END(Chain, 0, CalleePopBytes, InGlue, DL);
  if (!Ins.empty())
    InGlue = Chain.getValue(1);
 
  // Handle result values, copying them out of physregs into vregs that we
  // return.
  return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
                         InVals, /*IsThisReturn=*/false, SDValue());
}
 
// This is identical to the default implementation in ExpandDYNAMIC_STACKALLOC,
// except for applying the wave size scale to the increment amount.
SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl(
    SDValue Op, SelectionDAG &DAG) const {
  const MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
  SDValue Tmp1 = Op;
  SDValue Tmp2 = Op.getValue(1);
  SDValue Tmp3 = Op.getOperand(2);
  SDValue Chain = Tmp1.getOperand(0);
 
  Register SPReg = Info->getStackPtrOffsetReg();
 
  // Chain the dynamic stack allocation so that it doesn't modify the stack
  // pointer when other instructions are using the stack.
  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
 
  SDValue Size  = Tmp2.getOperand(1);
  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
  Chain = SP.getValue(1);
  MaybeAlign Alignment = cast<ConstantSDNode>(Tmp3)->getMaybeAlignValue();
  const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
  unsigned Opc =
    TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
    ISD::ADD : ISD::SUB;
 
  SDValue ScaledSize = DAG.getNode(
      ISD::SHL, dl, VT, Size,
      DAG.getConstant(Subtarget->getWavefrontSizeLog2(), dl, MVT::i32));
 
  Align StackAlign = TFL->getStackAlign();
  Tmp1 = DAG.getNode(Opc, dl, VT, SP, ScaledSize); // Value
  if (Alignment && *Alignment > StackAlign) {
    Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
                       DAG.getConstant(-(uint64_t)Alignment->value()
                                           << Subtarget->getWavefrontSizeLog2(),
                                       dl, VT));
  }
 
  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);    // Output chain
  Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
 
  return DAG.getMergeValues({Tmp1, Tmp2}, dl);
}
 
SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                  SelectionDAG &DAG) const {
  // We only handle constant sizes here to allow non-entry block, static sized
  // allocas. A truly dynamic value is more difficult to support because we
  // don't know if the size value is uniform or not. If the size isn't uniform,
  // we would need to do a wave reduction to get the maximum size to know how
  // much to increment the uniform stack pointer.
  SDValue Size = Op.getOperand(1);
  if (isa<ConstantSDNode>(Size))
      return lowerDYNAMIC_STACKALLOCImpl(Op, DAG); // Use "generic" expansion.
 
  return AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(Op, DAG);
}
 
SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
  if (Op.getValueType() != MVT::i32)
    return Op; // Defer to cannot select error.
 
  Register SP = getStackPointerRegisterToSaveRestore();
  SDLoc SL(Op);
 
  SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32);
 
  // Convert from wave uniform to swizzled vector address. This should protect
  // from any edge cases where the stacksave result isn't directly used with
  // stackrestore.
  SDValue VectorAddress =
      DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP);
  return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL);
}
 
SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
                                            SelectionDAG &DAG) const {
  SDLoc SL(Op);
  assert(Op.getValueType() == MVT::i32);
 
  uint32_t BothRoundHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 4);
  SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
 
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
  SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(),
                               Op.getOperand(0), IntrinID, GetRoundBothImm);
 
  // There are two rounding modes, one for f32 and one for f64/f16. We only
  // report in the standard value range if both are the same.
  //
  // The raw values also differ from the expected FLT_ROUNDS values. Nearest
  // ties away from zero is not supported, and the other values are rotated by
  // 1.
  //
  // If the two rounding modes are not the same, report a target defined value.
 
  // Mode register rounding mode fields:
  //
  // [1:0] Single-precision round mode.
  // [3:2] Double/Half-precision round mode.
  //
  // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
  //
  //             Hardware   Spec
  // Toward-0        3        0
  // Nearest Even    0        1
  // +Inf            1        2
  // -Inf            2        3
  //  NearestAway0  N/A       4
  //
  // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
  // table we can index by the raw hardware mode.
  //
  // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
 
  SDValue BitTable =
      DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64);
 
  SDValue Two = DAG.getConstant(2, SL, MVT::i32);
  SDValue RoundModeTimesNumBits =
      DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two);
 
  // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
  // knew only one mode was demanded.
  SDValue TableValue =
      DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
  SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
 
  SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32);
  SDValue TableEntry =
      DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask);
 
  // There's a gap in the 4-bit encoded table and actual enum values, so offset
  // if it's an extended value.
  SDValue Four = DAG.getConstant(4, SL, MVT::i32);
  SDValue IsStandardValue =
      DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT);
  SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four);
  SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue,
                               TableEntry, EnumOffset);
 
  return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL);
}
 
SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
                                            SelectionDAG &DAG) const {
  SDLoc SL(Op);
 
  SDValue NewMode = Op.getOperand(1);
  assert(NewMode.getValueType() == MVT::i32);
 
  // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
  // hardware MODE.fp_round values.
  if (auto *ConstMode = dyn_cast<ConstantSDNode>(NewMode)) {
    uint32_t ClampedVal = std::min(
        static_cast<uint32_t>(ConstMode->getZExtValue()),
        static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
    NewMode = DAG.getConstant(
        AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32);
  } else {
    // If we know the input can only be one of the supported standard modes in
    // the range 0-3, we can use a simplified mapping to hardware values.
    KnownBits KB = DAG.computeKnownBits(NewMode);
    const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
    // The supported standard values are 0-3. The extended values start at 8. We
    // need to offset by 4 if the value is in the extended range.
 
    if (UseReducedTable) {
      // Truncate to the low 32-bits.
      SDValue BitTable = DAG.getConstant(
          AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32);
 
      SDValue Two = DAG.getConstant(2, SL, MVT::i32);
      SDValue RoundModeTimesNumBits =
          DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two);
 
      NewMode =
          DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits);
 
      // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
      // the table extracted bits into inline immediates.
    } else {
      // table_index = umin(value, value - 4)
      // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
      SDValue BitTable =
          DAG.getConstant(AMDGPU::FltRoundToHWConversionTable, SL, MVT::i64);
 
      SDValue Four = DAG.getConstant(4, SL, MVT::i32);
      SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four);
      SDValue IndexVal =
          DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum);
 
      SDValue Two = DAG.getConstant(2, SL, MVT::i32);
      SDValue RoundModeTimesNumBits =
          DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two);
 
      SDValue TableValue =
          DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
      SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
 
      // No need to mask out the high bits since the setreg will ignore them
      // anyway.
      NewMode = TruncTable;
    }
 
    // Insert a readfirstlane in case the value is a VGPR. We could do this
    // earlier and keep more operations scalar, but that interferes with
    // combining the source.
    SDValue ReadFirstLaneID =
        DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
    NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                          ReadFirstLaneID, NewMode);
  }
 
  // N.B. The setreg will be later folded into s_round_mode on supported
  // targets.
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
  uint32_t BothRoundHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 4);
  SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
 
  SDValue SetReg =
      DAG.getNode(ISD::INTRINSIC_VOID, SL, Op->getVTList(), Op.getOperand(0),
                  IntrinID, RoundBothImm, NewMode);
 
  return SetReg;
}
 
SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
  if (Op->isDivergent())
    return SDValue();
 
  switch (cast<MemSDNode>(Op)->getAddressSpace()) {
  case AMDGPUAS::FLAT_ADDRESS:
  case AMDGPUAS::GLOBAL_ADDRESS:
  case AMDGPUAS::CONSTANT_ADDRESS:
  case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
    break;
  default:
    return SDValue();
  }
 
  return Op;
}
 
// Work around DAG legality rules only based on the result type.
SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
  bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
  SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
  EVT SrcVT = Src.getValueType();
 
  if (SrcVT.getScalarType() != MVT::bf16)
    return Op;
 
  SDLoc SL(Op);
  SDValue BitCast =
      DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
 
  EVT DstVT = Op.getValueType();
  if (IsStrict)
    llvm_unreachable("Need STRICT_BF16_TO_FP");
 
  return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
}
 
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
  SDLoc SL(Op);
  if (Op.getValueType() != MVT::i64)
    return Op;
 
  uint32_t ModeHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23);
  SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
  uint32_t TrapHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5);
  SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
 
  SDVTList VTList = DAG.getVTList(MVT::i32, MVT::Other);
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
  SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
                                   Op.getOperand(0), IntrinID, ModeHwRegImm);
  SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
                                   Op.getOperand(0), IntrinID, TrapHwRegImm);
  SDValue TokenReg =
      DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1),
                  GetTrapReg.getValue(1));
 
  SDValue CvtPtr =
      DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg);
  SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
 
  return DAG.getMergeValues({Result, TokenReg}, SL);
}
 
SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
  SDLoc SL(Op);
  if (Op.getOperand(1).getValueType() != MVT::i64)
    return Op;
 
  SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1));
  SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
                                   DAG.getConstant(0, SL, MVT::i32));
  SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
                                   DAG.getConstant(1, SL, MVT::i32));
 
  SDValue ReadFirstLaneID =
      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
  NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                           ReadFirstLaneID, NewModeReg);
  NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                           ReadFirstLaneID, NewTrapReg);
 
  unsigned ModeHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23);
  SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
  unsigned TrapHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5);
  SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
 
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
  SDValue SetModeReg =
      DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
                  IntrinID, ModeHwRegImm, NewModeReg);
  SDValue SetTrapReg =
      DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
                  IntrinID, TrapHwRegImm, NewTrapReg);
  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg);
}
 
Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT,
                                             const MachineFunction &MF) const {
  Register Reg = StringSwitch<Register>(RegName)
    .Case("m0", AMDGPU::M0)
    .Case("exec", AMDGPU::EXEC)
    .Case("exec_lo", AMDGPU::EXEC_LO)
    .Case("exec_hi", AMDGPU::EXEC_HI)
    .Case("flat_scratch", AMDGPU::FLAT_SCR)
    .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
    .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
    .Default(Register());
 
  if (Reg == AMDGPU::NoRegister) {
    report_fatal_error(Twine("invalid register name \""
                             + StringRef(RegName)  + "\"."));
 
  }
 
  if (!Subtarget->hasFlatScrRegister() &&
       Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) {
    report_fatal_error(Twine("invalid register \""
                             + StringRef(RegName)  + "\" for subtarget."));
  }
 
  switch (Reg) {
  case AMDGPU::M0:
  case AMDGPU::EXEC_LO:
  case AMDGPU::EXEC_HI:
  case AMDGPU::FLAT_SCR_LO:
  case AMDGPU::FLAT_SCR_HI:
    if (VT.getSizeInBits() == 32)