NVPTXISelLowering.cpp

Go to the documentation of this file.

//===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the interfaces that NVPTX uses to lower LLVM code into a
// selection DAG.
//
//===----------------------------------------------------------------------===//
 
#include "NVPTXISelLowering.h"
#include "MCTargetDesc/NVPTXBaseInfo.h"
#include "NVPTX.h"
#include "NVPTXSubtarget.h"
#include "NVPTXTargetMachine.h"
#include "NVPTXTargetObjectFile.h"
#include "NVPTXUtilities.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineValueType.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/FPEnv.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <iterator>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
 
#define DEBUG_TYPE "nvptx-lower"
 
using namespace llvm;
 
static std::atomic<unsigned> GlobalUniqueCallSite;
 
static cl::opt<bool> sched4reg(
    "nvptx-sched4reg",
    cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));
 
static cl::opt<unsigned> FMAContractLevelOpt(
    "nvptx-fma-level", cl::Hidden,
    cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
             " 1: do it  2: do it aggressively"),
    cl::init(2));
 
static cl::opt<int> UsePrecDivF32(
    "nvptx-prec-divf32", cl::Hidden,
    cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
             " IEEE Compliant F32 div.rnd if available."),
    cl::init(2));
 
static cl::opt<bool> UsePrecSqrtF32(
    "nvptx-prec-sqrtf32", cl::Hidden,
    cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
    cl::init(true));
 
static cl::opt<bool> ForceMinByValParamAlign(
    "nvptx-force-min-byval-param-align", cl::Hidden,
    cl::desc("NVPTX Specific: force 4-byte minimal alignment for byval"
             " params of device functions."),
    cl::init(false));
 
int NVPTXTargetLowering::getDivF32Level() const {
  if (UsePrecDivF32.getNumOccurrences() > 0) {
    // If nvptx-prec-div32=N is used on the command-line, always honor it
    return UsePrecDivF32;
  } else {
    // Otherwise, use div.approx if fast math is enabled
    if (getTargetMachine().Options.UnsafeFPMath)
      return 0;
    else
      return 2;
  }
}
 
bool NVPTXTargetLowering::usePrecSqrtF32() const {
  if (UsePrecSqrtF32.getNumOccurrences() > 0) {
    // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
    return UsePrecSqrtF32;
  } else {
    // Otherwise, use sqrt.approx if fast math is enabled
    return !getTargetMachine().Options.UnsafeFPMath;
  }
}
 
bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
  return MF.getDenormalMode(APFloat::IEEEsingle()).Output ==
         DenormalMode::PreserveSign;
}
 
static bool IsPTXVectorType(MVT VT) {
  switch (VT.SimpleTy) {
  default:
    return false;
  case MVT::v2i1:
  case MVT::v4i1:
  case MVT::v2i8:
  case MVT::v4i8:
  case MVT::v2i16:
  case MVT::v4i16:
  case MVT::v8i16: // <4 x i16x2>
  case MVT::v2i32:
  case MVT::v4i32:
  case MVT::v2i64:
  case MVT::v2f16:
  case MVT::v4f16:
  case MVT::v8f16: // <4 x f16x2>
  case MVT::v2bf16:
  case MVT::v4bf16:
  case MVT::v8bf16: // <4 x bf16x2>
  case MVT::v2f32:
  case MVT::v4f32:
  case MVT::v2f64:
    return true;
  }
}
 
static bool Is16bitsType(MVT VT) {
  return (VT.SimpleTy == MVT::f16 || VT.SimpleTy == MVT::bf16 ||
          VT.SimpleTy == MVT::i16);
}
 
/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
/// EVTs that compose it.  Unlike ComputeValueVTs, this will break apart vectors
/// into their primitive components.
/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
/// LowerCall, and LowerReturn.
static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
                               Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
                               SmallVectorImpl<uint64_t> *Offsets = nullptr,
                               uint64_t StartingOffset = 0) {
  SmallVector<EVT, 16> TempVTs;
  SmallVector<uint64_t, 16> TempOffsets;
 
  // Special case for i128 - decompose to (i64, i64)
  if (Ty->isIntegerTy(128)) {
    ValueVTs.push_back(EVT(MVT::i64));
    ValueVTs.push_back(EVT(MVT::i64));
 
    if (Offsets) {
      Offsets->push_back(StartingOffset + 0);
      Offsets->push_back(StartingOffset + 8);
    }
 
    return;
  }
 
  // Given a struct type, recursively traverse the elements with custom ComputePTXValueVTs.
  if (StructType *STy = dyn_cast<StructType>(Ty)) {
    auto const *SL = DL.getStructLayout(STy);
    auto ElementNum = 0;
    for(auto *EI : STy->elements()) {
      ComputePTXValueVTs(TLI, DL, EI, ValueVTs, Offsets,
                         StartingOffset + SL->getElementOffset(ElementNum));
      ++ElementNum;
    }
    return;
  }
 
  ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset);
  for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {
    EVT VT = TempVTs[i];
    uint64_t Off = TempOffsets[i];
    // Split vectors into individual elements, except for v2f16, which
    // we will pass as a single scalar.
    if (VT.isVector()) {
      unsigned NumElts = VT.getVectorNumElements();
      EVT EltVT = VT.getVectorElementType();
      // Vectors with an even number of f16 elements will be passed to
      // us as an array of v2f16/v2bf16 elements. We must match this so we
      // stay in sync with Ins/Outs.
      if ((Is16bitsType(EltVT.getSimpleVT())) && NumElts % 2 == 0) {
        switch (EltVT.getSimpleVT().SimpleTy) {
        case MVT::f16:
          EltVT = MVT::v2f16;
          break;
        case MVT::bf16:
          EltVT = MVT::v2bf16;
          break;
        case MVT::i16:
          EltVT = MVT::v2i16;
          break;
        default:
          llvm_unreachable("Unexpected type");
        }
        NumElts /= 2;
      }
      for (unsigned j = 0; j != NumElts; ++j) {
        ValueVTs.push_back(EltVT);
        if (Offsets)
          Offsets->push_back(Off + j * EltVT.getStoreSize());
      }
    } else {
      ValueVTs.push_back(VT);
      if (Offsets)
        Offsets->push_back(Off);
    }
  }
}
 
/// PromoteScalarIntegerPTX
/// Used to make sure the arguments/returns are suitable for passing
/// and promote them to a larger size if they're not.
///
/// The promoted type is placed in \p PromoteVT if the function returns true.
static bool PromoteScalarIntegerPTX(const EVT &VT, MVT *PromotedVT) {
  if (VT.isScalarInteger()) {
    switch (PowerOf2Ceil(VT.getFixedSizeInBits())) {
    default:
      llvm_unreachable(
          "Promotion is not suitable for scalars of size larger than 64-bits");
    case 1:
      *PromotedVT = MVT::i1;
      break;
    case 2:
    case 4:
    case 8:
      *PromotedVT = MVT::i8;
      break;
    case 16:
      *PromotedVT = MVT::i16;
      break;
    case 32:
      *PromotedVT = MVT::i32;
      break;
    case 64:
      *PromotedVT = MVT::i64;
      break;
    }
    return EVT(*PromotedVT) != VT;
  }
  return false;
}
 
// Check whether we can merge loads/stores of some of the pieces of a
// flattened function parameter or return value into a single vector
// load/store.
//
// The flattened parameter is represented as a list of EVTs and
// offsets, and the whole structure is aligned to ParamAlignment. This
// function determines whether we can load/store pieces of the
// parameter starting at index Idx using a single vectorized op of
// size AccessSize. If so, it returns the number of param pieces
// covered by the vector op. Otherwise, it returns 1.
static unsigned CanMergeParamLoadStoresStartingAt(
    unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
    const SmallVectorImpl<uint64_t> &Offsets, Align ParamAlignment) {
 
  // Can't vectorize if param alignment is not sufficient.
  if (ParamAlignment < AccessSize)
    return 1;
  // Can't vectorize if offset is not aligned.
  if (Offsets[Idx] & (AccessSize - 1))
    return 1;
 
  EVT EltVT = ValueVTs[Idx];
  unsigned EltSize = EltVT.getStoreSize();
 
  // Element is too large to vectorize.
  if (EltSize >= AccessSize)
    return 1;
 
  unsigned NumElts = AccessSize / EltSize;
  // Can't vectorize if AccessBytes if not a multiple of EltSize.
  if (AccessSize != EltSize * NumElts)
    return 1;
 
  // We don't have enough elements to vectorize.
  if (Idx + NumElts > ValueVTs.size())
    return 1;
 
  // PTX ISA can only deal with 2- and 4-element vector ops.
  if (NumElts != 4 && NumElts != 2)
    return 1;
 
  for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) {
    // Types do not match.
    if (ValueVTs[j] != EltVT)
      return 1;
 
    // Elements are not contiguous.
    if (Offsets[j] - Offsets[j - 1] != EltSize)
      return 1;
  }
  // OK. We can vectorize ValueVTs[i..i+NumElts)
  return NumElts;
}
 
// Flags for tracking per-element vectorization state of loads/stores
// of a flattened function parameter or return value.
enum ParamVectorizationFlags {
  PVF_INNER = 0x0, // Middle elements of a vector.
  PVF_FIRST = 0x1, // First element of the vector.
  PVF_LAST = 0x2,  // Last element of the vector.
  // Scalar is effectively a 1-element vector.
  PVF_SCALAR = PVF_FIRST | PVF_LAST
};
 
// Computes whether and how we can vectorize the loads/stores of a
// flattened function parameter or return value.
//
// The flattened parameter is represented as the list of ValueVTs and
// Offsets, and is aligned to ParamAlignment bytes. We return a vector
// of the same size as ValueVTs indicating how each piece should be
// loaded/stored (i.e. as a scalar, or as part of a vector
// load/store).
static SmallVector<ParamVectorizationFlags, 16>
VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
                     const SmallVectorImpl<uint64_t> &Offsets,
                     Align ParamAlignment, bool IsVAArg = false) {
  // Set vector size to match ValueVTs and mark all elements as
  // scalars by default.
  SmallVector<ParamVectorizationFlags, 16> VectorInfo;
  VectorInfo.assign(ValueVTs.size(), PVF_SCALAR);
 
  if (IsVAArg)
    return VectorInfo;
 
  // Check what we can vectorize using 128/64/32-bit accesses.
  for (int I = 0, E = ValueVTs.size(); I != E; ++I) {
    // Skip elements we've already processed.
    assert(VectorInfo[I] == PVF_SCALAR && "Unexpected vector info state.");
    for (unsigned AccessSize : {16, 8, 4, 2}) {
      unsigned NumElts = CanMergeParamLoadStoresStartingAt(
          I, AccessSize, ValueVTs, Offsets, ParamAlignment);
      // Mark vectorized elements.
      switch (NumElts) {
      default:
        llvm_unreachable("Unexpected return value");
      case 1:
        // Can't vectorize using this size, try next smaller size.
        continue;
      case 2:
        assert(I + 1 < E && "Not enough elements.");
        VectorInfo[I] = PVF_FIRST;
        VectorInfo[I + 1] = PVF_LAST;
        I += 1;
        break;
      case 4:
        assert(I + 3 < E && "Not enough elements.");
        VectorInfo[I] = PVF_FIRST;
        VectorInfo[I + 1] = PVF_INNER;
        VectorInfo[I + 2] = PVF_INNER;
        VectorInfo[I + 3] = PVF_LAST;
        I += 3;
        break;
      }
      // Break out of the inner loop because we've already succeeded
      // using largest possible AccessSize.
      break;
    }
  }
  return VectorInfo;
}
 
// NVPTXTargetLowering Constructor.
NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
                                         const NVPTXSubtarget &STI)
    : TargetLowering(TM), nvTM(&TM), STI(STI) {
  // always lower memset, memcpy, and memmove intrinsics to load/store
  // instructions, rather
  // then generating calls to memset, mempcy or memmove.
  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = (unsigned)0xFFFFFFFF;
  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = (unsigned) 0xFFFFFFFF;
  MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = (unsigned) 0xFFFFFFFF;
 
  setBooleanContents(ZeroOrNegativeOneBooleanContent);
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
  // Jump is Expensive. Don't create extra control flow for 'and', 'or'
  // condition branches.
  setJumpIsExpensive(true);
 
  // Wide divides are _very_ slow. Try to reduce the width of the divide if
  // possible.
  addBypassSlowDiv(64, 32);
 
  // By default, use the Source scheduling
  if (sched4reg)
    setSchedulingPreference(Sched::RegPressure);
  else
    setSchedulingPreference(Sched::Source);
 
  auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                    LegalizeAction NoF16Action) {
    setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action);
  };
 
  auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                    LegalizeAction NoBF16Action) {
    bool IsOpSupported = STI.hasBF16Math();
    // Few instructions are available on sm_90 only
    switch(Op) {
      case ISD::FADD:
      case ISD::FMUL:
      case ISD::FSUB:
        IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
        break;
    }
    setOperationAction(
        Op, VT, IsOpSupported ? Action : NoBF16Action);
  };
 
  auto setI16x2OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoI16x2Action) {
    bool IsOpSupported = false;
    // instructions are available on sm_90 only
    switch (Op) {
    case ISD::ADD:
    case ISD::SMAX:
    case ISD::SMIN:
    case ISD::UMIN:
    case ISD::UMAX:
    case ISD::SUB:
      IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 80;
      break;
    }
    setOperationAction(Op, VT, IsOpSupported ? Action : NoI16x2Action);
  };
 
  addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass);
  addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass);
  addRegisterClass(MVT::v2i16, &NVPTX::Int32RegsRegClass);
  addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass);
  addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass);
  addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass);
  addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass);
  addRegisterClass(MVT::f16, &NVPTX::Int16RegsRegClass);
  addRegisterClass(MVT::v2f16, &NVPTX::Int32RegsRegClass);
  addRegisterClass(MVT::bf16, &NVPTX::Int16RegsRegClass);
  addRegisterClass(MVT::v2bf16, &NVPTX::Int32RegsRegClass);
 
  // Conversion to/from FP16/FP16x2 is always legal.
  setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Expand);
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f16, Expand);
 
  setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote);
  setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand);
 
  // Conversion to/from BFP16/BFP16x2 is always legal.
  setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Custom);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2bf16, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2bf16, Expand);
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2bf16, Expand);
 
  setBF16OperationAction(ISD::SETCC, MVT::bf16, Legal, Promote);
  setBF16OperationAction(ISD::SETCC, MVT::v2bf16, Legal, Expand);
 
  // Conversion to/from i16/i16x2 is always legal.
  setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Expand);
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i16, Expand);
 
  // Operations not directly supported by NVPTX.
  for (MVT VT :
       {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32, MVT::f64,
        MVT::i1, MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64}) {
    setOperationAction(ISD::SELECT_CC, VT, Expand);
    setOperationAction(ISD::BR_CC, VT, Expand);
  }
 
  // Some SIGN_EXTEND_INREG can be done using cvt instruction.
  // For others we will expand to a SHL/SRA pair.
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand);
 
  setOperationAction(ISD::SHL_PARTS, MVT::i32  , Custom);
  setOperationAction(ISD::SRA_PARTS, MVT::i32  , Custom);
  setOperationAction(ISD::SRL_PARTS, MVT::i32  , Custom);
  setOperationAction(ISD::SHL_PARTS, MVT::i64  , Custom);
  setOperationAction(ISD::SRA_PARTS, MVT::i64  , Custom);
  setOperationAction(ISD::SRL_PARTS, MVT::i64  , Custom);
 
  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
  setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
 
  // TODO: we may consider expanding ROTL/ROTR on older GPUs.  Currently on GPUs
  // that don't have h/w rotation we lower them to multi-instruction assembly.
  // See ROT*_sw in NVPTXIntrInfo.td
  setOperationAction(ISD::ROTL, MVT::i64, Legal);
  setOperationAction(ISD::ROTR, MVT::i64, Legal);
  setOperationAction(ISD::ROTL, MVT::i32, Legal);
  setOperationAction(ISD::ROTR, MVT::i32, Legal);
 
  setOperationAction(ISD::ROTL, MVT::i16, Expand);
  setOperationAction(ISD::ROTL, MVT::v2i16, Expand);
  setOperationAction(ISD::ROTR, MVT::i16, Expand);
  setOperationAction(ISD::ROTR, MVT::v2i16, Expand);
  setOperationAction(ISD::ROTL, MVT::i8, Expand);
  setOperationAction(ISD::ROTR, MVT::i8, Expand);
  setOperationAction(ISD::BSWAP, MVT::i16, Expand);
  setOperationAction(ISD::BSWAP, MVT::v2i16, Expand);
  setOperationAction(ISD::BSWAP, MVT::i32, Expand);
  setOperationAction(ISD::BSWAP, MVT::i64, Expand);
 
  // Indirect branch is not supported.
  // This also disables Jump Table creation.
  setOperationAction(ISD::BR_JT, MVT::Other, Expand);
  setOperationAction(ISD::BRIND, MVT::Other, Expand);
 
  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
 
  // We want to legalize constant related memmove and memcopy
  // intrinsics.
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
 
  // Turn FP extload into load/fpextend
  setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2bf16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2bf16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4bf16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4bf16, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
  // Turn FP truncstore into trunc + store.
  // FIXME: vector types should also be expanded
  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
  setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
  setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
  // PTX does not support load / store predicate registers
  setOperationAction(ISD::LOAD, MVT::i1, Custom);
  setOperationAction(ISD::STORE, MVT::i1, Custom);
 
  for (MVT VT : MVT::integer_valuetypes()) {
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
    setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
    setTruncStoreAction(VT, MVT::i1, Expand);
  }
 
  // expand extload of vector of integers.
  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::v2i16,
                   MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
 
  // This is legal in NVPTX
  setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
  setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
  setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
  setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);
 
  // TRAP can be lowered to PTX trap
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
  // Register custom handling for vector loads/stores
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    if (IsPTXVectorType(VT)) {
      setOperationAction(ISD::LOAD, VT, Custom);
      setOperationAction(ISD::STORE, VT, Custom);
      setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);
    }
  }
 
  // Support varargs.
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
  setOperationAction(ISD::VAARG, MVT::Other, Custom);
  setOperationAction(ISD::VACOPY, MVT::Other, Expand);
  setOperationAction(ISD::VAEND, MVT::Other, Expand);
 
  // Custom handling for i8 intrinsics
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
 
  for (const auto& Ty : {MVT::i16, MVT::i32, MVT::i64}) {
    setOperationAction(ISD::ABS,  Ty, Legal);
    setOperationAction(ISD::SMIN, Ty, Legal);
    setOperationAction(ISD::SMAX, Ty, Legal);
    setOperationAction(ISD::UMIN, Ty, Legal);
    setOperationAction(ISD::UMAX, Ty, Legal);
 
    setOperationAction(ISD::CTPOP, Ty, Legal);
    setOperationAction(ISD::CTLZ, Ty, Legal);
  }
 
  setI16x2OperationAction(ISD::ABS, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::SMIN, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::SMAX, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::UMIN, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::UMAX, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::CTPOP, MVT::v2i16, Legal, Expand);
  setI16x2OperationAction(ISD::CTLZ, MVT::v2i16, Legal, Expand);
 
  setI16x2OperationAction(ISD::ADD, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::SUB, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::MUL, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::SHL, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::SREM, MVT::v2i16, Legal, Custom);
  setI16x2OperationAction(ISD::UREM, MVT::v2i16, Legal, Custom);
 
  // Other arithmetic and logic ops are unsupported.
  setOperationAction({ISD::AND, ISD::OR, ISD::XOR, ISD::SDIV, ISD::UDIV,
                      ISD::SRA, ISD::SRL, ISD::MULHS, ISD::MULHU,
                      ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
                      ISD::UINT_TO_FP},
                     MVT::v2i16, Expand);
 
  setOperationAction(ISD::ADDC, MVT::i32, Legal);
  setOperationAction(ISD::ADDE, MVT::i32, Legal);
  setOperationAction(ISD::SUBC, MVT::i32, Legal);
  setOperationAction(ISD::SUBE, MVT::i32, Legal);
  if (STI.getPTXVersion() >= 43) {
    setOperationAction(ISD::ADDC, MVT::i64, Legal);
    setOperationAction(ISD::ADDE, MVT::i64, Legal);
    setOperationAction(ISD::SUBC, MVT::i64, Legal);
    setOperationAction(ISD::SUBE, MVT::i64, Legal);
  }
 
  setOperationAction(ISD::CTTZ, MVT::i16, Expand);
  setOperationAction(ISD::CTTZ, MVT::v2i16, Expand);
  setOperationAction(ISD::CTTZ, MVT::i32, Expand);
  setOperationAction(ISD::CTTZ, MVT::i64, Expand);
 
  // PTX does not directly support SELP of i1, so promote to i32 first
  setOperationAction(ISD::SELECT, MVT::i1, Custom);
 
  // PTX cannot multiply two i64s in a single instruction.
  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
 
  // We have some custom DAG combine patterns for these nodes
  setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::FADD, ISD::MUL, ISD::SHL,
                       ISD::SREM, ISD::UREM, ISD::EXTRACT_VECTOR_ELT});
 
  // setcc for f16x2 and bf16x2 needs special handling to prevent
  // legalizer's attempt to scalarize it due to v2i1 not being legal.
  if (STI.allowFP16Math() || STI.hasBF16Math())
    setTargetDAGCombine(ISD::SETCC);
 
  // Promote fp16 arithmetic if fp16 hardware isn't available or the
  // user passed --nvptx-no-fp16-math. The flag is useful because,
  // although sm_53+ GPUs have some sort of FP16 support in
  // hardware, only sm_53 and sm_60 have full implementation. Others
  // only have token amount of hardware and are likely to run faster
  // by using fp32 units instead.
  for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) {
    setFP16OperationAction(Op, MVT::f16, Legal, Promote);
    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
    setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
    setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
    // bf16 must be promoted to f32.
    if (getOperationAction(Op, MVT::bf16) == Promote)
      AddPromotedToType(Op, MVT::bf16, MVT::f32);
  }
 
  // f16/f16x2 neg was introduced in PTX 60, SM_53.
  const bool IsFP16FP16x2NegAvailable = STI.getSmVersion() >= 53 &&
                                        STI.getPTXVersion() >= 60 &&
                                        STI.allowFP16Math();
  for (const auto &VT : {MVT::f16, MVT::v2f16})
    setOperationAction(ISD::FNEG, VT,
                       IsFP16FP16x2NegAvailable ? Legal : Expand);
 
  setBF16OperationAction(ISD::FNEG, MVT::bf16, Legal, Expand);
  setBF16OperationAction(ISD::FNEG, MVT::v2bf16, Legal, Expand);
  // (would be) Library functions.
 
  // These map to conversion instructions for scalar FP types.
  for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
                         ISD::FROUNDEVEN, ISD::FTRUNC}) {
    setOperationAction(Op, MVT::bf16, Legal);
    setOperationAction(Op, MVT::f16, Legal);
    setOperationAction(Op, MVT::f32, Legal);
    setOperationAction(Op, MVT::f64, Legal);
    setOperationAction(Op, MVT::v2f16, Expand);
    setOperationAction(Op, MVT::v2bf16, Expand);
  }
 
  setOperationAction(ISD::FROUND, MVT::f16, Promote);
  setOperationAction(ISD::FROUND, MVT::v2f16, Expand);
  setOperationAction(ISD::FROUND, MVT::bf16, Promote);
  setOperationAction(ISD::FROUND, MVT::v2bf16, Expand);
  setOperationAction(ISD::FROUND, MVT::f32, Custom);
  setOperationAction(ISD::FROUND, MVT::f64, Custom);
 
 
  // 'Expand' implements FCOPYSIGN without calling an external library.
  setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::v2f16, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::v2bf16, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
 
  // These map to corresponding instructions for f32/f64. f16 must be
  // promoted to f32. v2f16 is expanded to f16, which is then promoted
  // to f32.
  for (const auto &Op :
       {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FABS}) {
    setOperationAction(Op, MVT::f16, Promote);
    setOperationAction(Op, MVT::bf16, Promote);
    setOperationAction(Op, MVT::f32, Legal);
    setOperationAction(Op, MVT::f64, Legal);
    setOperationAction(Op, MVT::v2f16, Expand);
    setOperationAction(Op, MVT::v2bf16, Expand);
  }
  // max.f16, max.f16x2 and max.NaN are supported on sm_80+.
  auto GetMinMaxAction = [&](LegalizeAction NotSm80Action) {
    bool IsAtLeastSm80 = STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
    return IsAtLeastSm80 ? Legal : NotSm80Action;
  };
  for (const auto &Op : {ISD::FMINNUM, ISD::FMAXNUM}) {
    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Promote), Promote);
    setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
    setOperationAction(Op, MVT::f32, Legal);
    setOperationAction(Op, MVT::f64, Legal);
    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
    setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
  }
  for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Expand), Expand);
    setFP16OperationAction(Op, MVT::bf16, Legal, Expand);
    setOperationAction(Op, MVT::f32, GetMinMaxAction(Expand));
    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
    setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
  }
 
  // No FEXP2, FLOG2.  The PTX ex2 and log2 functions are always approximate.
  // No FPOW or FREM in PTX.
 
  // Now deduce the information based on the above mentioned
  // actions
  computeRegisterProperties(STI.getRegisterInfo());
 
  setMinCmpXchgSizeInBits(32);
}
 
const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
  switch ((NVPTXISD::NodeType)Opcode) {
  case NVPTXISD::FIRST_NUMBER:
    break;
  case NVPTXISD::CALL:
    return "NVPTXISD::CALL";
  case NVPTXISD::RET_GLUE:
    return "NVPTXISD::RET_GLUE";
  case NVPTXISD::LOAD_PARAM:
    return "NVPTXISD::LOAD_PARAM";
  case NVPTXISD::Wrapper:
    return "NVPTXISD::Wrapper";
  case NVPTXISD::DeclareParam:
    return "NVPTXISD::DeclareParam";
  case NVPTXISD::DeclareScalarParam:
    return "NVPTXISD::DeclareScalarParam";
  case NVPTXISD::DeclareRet:
    return "NVPTXISD::DeclareRet";
  case NVPTXISD::DeclareScalarRet:
    return "NVPTXISD::DeclareScalarRet";
  case NVPTXISD::DeclareRetParam:
    return "NVPTXISD::DeclareRetParam";
  case NVPTXISD::PrintCall:
    return "NVPTXISD::PrintCall";
  case NVPTXISD::PrintConvergentCall:
    return "NVPTXISD::PrintConvergentCall";
  case NVPTXISD::PrintCallUni:
    return "NVPTXISD::PrintCallUni";
  case NVPTXISD::PrintConvergentCallUni:
    return "NVPTXISD::PrintConvergentCallUni";
  case NVPTXISD::LoadParam:
    return "NVPTXISD::LoadParam";
  case NVPTXISD::LoadParamV2:
    return "NVPTXISD::LoadParamV2";
  case NVPTXISD::LoadParamV4:
    return "NVPTXISD::LoadParamV4";
  case NVPTXISD::StoreParam:
    return "NVPTXISD::StoreParam";
  case NVPTXISD::StoreParamV2:
    return "NVPTXISD::StoreParamV2";
  case NVPTXISD::StoreParamV4:
    return "NVPTXISD::StoreParamV4";
  case NVPTXISD::StoreParamS32:
    return "NVPTXISD::StoreParamS32";
  case NVPTXISD::StoreParamU32:
    return "NVPTXISD::StoreParamU32";
  case NVPTXISD::CallArgBegin:
    return "NVPTXISD::CallArgBegin";
  case NVPTXISD::CallArg:
    return "NVPTXISD::CallArg";
  case NVPTXISD::LastCallArg:
    return "NVPTXISD::LastCallArg";
  case NVPTXISD::CallArgEnd:
    return "NVPTXISD::CallArgEnd";
  case NVPTXISD::CallVoid:
    return "NVPTXISD::CallVoid";
  case NVPTXISD::CallVal:
    return "NVPTXISD::CallVal";
  case NVPTXISD::CallSymbol:
    return "NVPTXISD::CallSymbol";
  case NVPTXISD::Prototype:
    return "NVPTXISD::Prototype";
  case NVPTXISD::MoveParam:
    return "NVPTXISD::MoveParam";
  case NVPTXISD::StoreRetval:
    return "NVPTXISD::StoreRetval";
  case NVPTXISD::StoreRetvalV2:
    return "NVPTXISD::StoreRetvalV2";
  case NVPTXISD::StoreRetvalV4:
    return "NVPTXISD::StoreRetvalV4";
  case NVPTXISD::PseudoUseParam:
    return "NVPTXISD::PseudoUseParam";
  case NVPTXISD::RETURN:
    return "NVPTXISD::RETURN";
  case NVPTXISD::CallSeqBegin:
    return "NVPTXISD::CallSeqBegin";
  case NVPTXISD::CallSeqEnd:
    return "NVPTXISD::CallSeqEnd";
  case NVPTXISD::CallPrototype:
    return "NVPTXISD::CallPrototype";
  case NVPTXISD::ProxyReg:
    return "NVPTXISD::ProxyReg";
  case NVPTXISD::LoadV2:
    return "NVPTXISD::LoadV2";
  case NVPTXISD::LoadV4:
    return "NVPTXISD::LoadV4";
  case NVPTXISD::LDGV2:
    return "NVPTXISD::LDGV2";
  case NVPTXISD::LDGV4:
    return "NVPTXISD::LDGV4";
  case NVPTXISD::LDUV2:
    return "NVPTXISD::LDUV2";
  case NVPTXISD::LDUV4:
    return "NVPTXISD::LDUV4";
  case NVPTXISD::StoreV2:
    return "NVPTXISD::StoreV2";
  case NVPTXISD::StoreV4:
    return "NVPTXISD::StoreV4";
  case NVPTXISD::FUN_SHFL_CLAMP:
    return "NVPTXISD::FUN_SHFL_CLAMP";
  case NVPTXISD::FUN_SHFR_CLAMP:
    return "NVPTXISD::FUN_SHFR_CLAMP";
  case NVPTXISD::IMAD:
    return "NVPTXISD::IMAD";
  case NVPTXISD::SETP_F16X2:
    return "NVPTXISD::SETP_F16X2";
  case NVPTXISD::Dummy:
    return "NVPTXISD::Dummy";
  case NVPTXISD::MUL_WIDE_SIGNED:
    return "NVPTXISD::MUL_WIDE_SIGNED";
  case NVPTXISD::MUL_WIDE_UNSIGNED:
    return "NVPTXISD::MUL_WIDE_UNSIGNED";
  case NVPTXISD::Tex1DFloatS32:        return "NVPTXISD::Tex1DFloatS32";
  case NVPTXISD::Tex1DFloatFloat:      return "NVPTXISD::Tex1DFloatFloat";
  case NVPTXISD::Tex1DFloatFloatLevel:
    return "NVPTXISD::Tex1DFloatFloatLevel";
  case NVPTXISD::Tex1DFloatFloatGrad:
    return "NVPTXISD::Tex1DFloatFloatGrad";
  case NVPTXISD::Tex1DS32S32:          return "NVPTXISD::Tex1DS32S32";
  case NVPTXISD::Tex1DS32Float:        return "NVPTXISD::Tex1DS32Float";
  case NVPTXISD::Tex1DS32FloatLevel:
    return "NVPTXISD::Tex1DS32FloatLevel";
  case NVPTXISD::Tex1DS32FloatGrad:
    return "NVPTXISD::Tex1DS32FloatGrad";
  case NVPTXISD::Tex1DU32S32:          return "NVPTXISD::Tex1DU32S32";
  case NVPTXISD::Tex1DU32Float:        return "NVPTXISD::Tex1DU32Float";
  case NVPTXISD::Tex1DU32FloatLevel:
    return "NVPTXISD::Tex1DU32FloatLevel";
  case NVPTXISD::Tex1DU32FloatGrad:
    return "NVPTXISD::Tex1DU32FloatGrad";
  case NVPTXISD::Tex1DArrayFloatS32:   return "NVPTXISD::Tex1DArrayFloatS32";
  case NVPTXISD::Tex1DArrayFloatFloat: return "NVPTXISD::Tex1DArrayFloatFloat";
  case NVPTXISD::Tex1DArrayFloatFloatLevel:
    return "NVPTXISD::Tex1DArrayFloatFloatLevel";
  case NVPTXISD::Tex1DArrayFloatFloatGrad:
    return "NVPTXISD::Tex1DArrayFloatFloatGrad";
  case NVPTXISD::Tex1DArrayS32S32:     return "NVPTXISD::Tex1DArrayS32S32";
  case NVPTXISD::Tex1DArrayS32Float:   return "NVPTXISD::Tex1DArrayS32Float";
  case NVPTXISD::Tex1DArrayS32FloatLevel:
    return "NVPTXISD::Tex1DArrayS32FloatLevel";
  case NVPTXISD::Tex1DArrayS32FloatGrad:
    return "NVPTXISD::Tex1DArrayS32FloatGrad";
  case NVPTXISD::Tex1DArrayU32S32:     return "NVPTXISD::Tex1DArrayU32S32";
  case NVPTXISD::Tex1DArrayU32Float:   return "NVPTXISD::Tex1DArrayU32Float";
  case NVPTXISD::Tex1DArrayU32FloatLevel:
    return "NVPTXISD::Tex1DArrayU32FloatLevel";
  case NVPTXISD::Tex1DArrayU32FloatGrad:
    return "NVPTXISD::Tex1DArrayU32FloatGrad";
  case NVPTXISD::Tex2DFloatS32:        return "NVPTXISD::Tex2DFloatS32";
  case NVPTXISD::Tex2DFloatFloat:      return "NVPTXISD::Tex2DFloatFloat";
  case NVPTXISD::Tex2DFloatFloatLevel:
    return "NVPTXISD::Tex2DFloatFloatLevel";
  case NVPTXISD::Tex2DFloatFloatGrad:
    return "NVPTXISD::Tex2DFloatFloatGrad";
  case NVPTXISD::Tex2DS32S32:          return "NVPTXISD::Tex2DS32S32";
  case NVPTXISD::Tex2DS32Float:        return "NVPTXISD::Tex2DS32Float";
  case NVPTXISD::Tex2DS32FloatLevel:
    return "NVPTXISD::Tex2DS32FloatLevel";
  case NVPTXISD::Tex2DS32FloatGrad:
    return "NVPTXISD::Tex2DS32FloatGrad";
  case NVPTXISD::Tex2DU32S32:          return "NVPTXISD::Tex2DU32S32";
  case NVPTXISD::Tex2DU32Float:        return "NVPTXISD::Tex2DU32Float";
  case NVPTXISD::Tex2DU32FloatLevel:
    return "NVPTXISD::Tex2DU32FloatLevel";
  case NVPTXISD::Tex2DU32FloatGrad:
    return "NVPTXISD::Tex2DU32FloatGrad";
  case NVPTXISD::Tex2DArrayFloatS32:   return "NVPTXISD::Tex2DArrayFloatS32";
  case NVPTXISD::Tex2DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat";
  case NVPTXISD::Tex2DArrayFloatFloatLevel:
    return "NVPTXISD::Tex2DArrayFloatFloatLevel";
  case NVPTXISD::Tex2DArrayFloatFloatGrad:
    return "NVPTXISD::Tex2DArrayFloatFloatGrad";
  case NVPTXISD::Tex2DArrayS32S32:     return "NVPTXISD::Tex2DArrayS32S32";
  case NVPTXISD::Tex2DArrayS32Float:   return "NVPTXISD::Tex2DArrayS32Float";
  case NVPTXISD::Tex2DArrayS32FloatLevel:
    return "NVPTXISD::Tex2DArrayS32FloatLevel";
  case NVPTXISD::Tex2DArrayS32FloatGrad:
    return "NVPTXISD::Tex2DArrayS32FloatGrad";
  case NVPTXISD::Tex2DArrayU32S32:     return "NVPTXISD::Tex2DArrayU32S32";
  case NVPTXISD::Tex2DArrayU32Float:   return "NVPTXISD::Tex2DArrayU32Float";
  case NVPTXISD::Tex2DArrayU32FloatLevel:
    return "NVPTXISD::Tex2DArrayU32FloatLevel";
  case NVPTXISD::Tex2DArrayU32FloatGrad:
    return "NVPTXISD::Tex2DArrayU32FloatGrad";
  case NVPTXISD::Tex3DFloatS32:        return "NVPTXISD::Tex3DFloatS32";
  case NVPTXISD::Tex3DFloatFloat:      return "NVPTXISD::Tex3DFloatFloat";
  case NVPTXISD::Tex3DFloatFloatLevel:
    return "NVPTXISD::Tex3DFloatFloatLevel";
  case NVPTXISD::Tex3DFloatFloatGrad:
    return "NVPTXISD::Tex3DFloatFloatGrad";
  case NVPTXISD::Tex3DS32S32:          return "NVPTXISD::Tex3DS32S32";
  case NVPTXISD::Tex3DS32Float:        return "NVPTXISD::Tex3DS32Float";
  case NVPTXISD::Tex3DS32FloatLevel:
    return "NVPTXISD::Tex3DS32FloatLevel";
  case NVPTXISD::Tex3DS32FloatGrad:
    return "NVPTXISD::Tex3DS32FloatGrad";
  case NVPTXISD::Tex3DU32S32:          return "NVPTXISD::Tex3DU32S32";
  case NVPTXISD::Tex3DU32Float:        return "NVPTXISD::Tex3DU32Float";
  case NVPTXISD::Tex3DU32FloatLevel:
    return "NVPTXISD::Tex3DU32FloatLevel";
  case NVPTXISD::Tex3DU32FloatGrad:
    return "NVPTXISD::Tex3DU32FloatGrad";
  case NVPTXISD::TexCubeFloatFloat:      return "NVPTXISD::TexCubeFloatFloat";
  case NVPTXISD::TexCubeFloatFloatLevel:
    return "NVPTXISD::TexCubeFloatFloatLevel";
  case NVPTXISD::TexCubeS32Float:        return "NVPTXISD::TexCubeS32Float";
  case NVPTXISD::TexCubeS32FloatLevel:
    return "NVPTXISD::TexCubeS32FloatLevel";
  case NVPTXISD::TexCubeU32Float:        return "NVPTXISD::TexCubeU32Float";
  case NVPTXISD::TexCubeU32FloatLevel:
    return "NVPTXISD::TexCubeU32FloatLevel";
  case NVPTXISD::TexCubeArrayFloatFloat:
    return "NVPTXISD::TexCubeArrayFloatFloat";
  case NVPTXISD::TexCubeArrayFloatFloatLevel:
    return "NVPTXISD::TexCubeArrayFloatFloatLevel";
  case NVPTXISD::TexCubeArrayS32Float:
    return "NVPTXISD::TexCubeArrayS32Float";
  case NVPTXISD::TexCubeArrayS32FloatLevel:
    return "NVPTXISD::TexCubeArrayS32FloatLevel";
  case NVPTXISD::TexCubeArrayU32Float:
    return "NVPTXISD::TexCubeArrayU32Float";
  case NVPTXISD::TexCubeArrayU32FloatLevel:
    return "NVPTXISD::TexCubeArrayU32FloatLevel";
  case NVPTXISD::Tld4R2DFloatFloat:
    return "NVPTXISD::Tld4R2DFloatFloat";
  case NVPTXISD::Tld4G2DFloatFloat:
    return "NVPTXISD::Tld4G2DFloatFloat";
  case NVPTXISD::Tld4B2DFloatFloat:
    return "NVPTXISD::Tld4B2DFloatFloat";
  case NVPTXISD::Tld4A2DFloatFloat:
    return "NVPTXISD::Tld4A2DFloatFloat";
  case NVPTXISD::Tld4R2DS64Float:
    return "NVPTXISD::Tld4R2DS64Float";
  case NVPTXISD::Tld4G2DS64Float:
    return "NVPTXISD::Tld4G2DS64Float";
  case NVPTXISD::Tld4B2DS64Float:
    return "NVPTXISD::Tld4B2DS64Float";
  case NVPTXISD::Tld4A2DS64Float:
    return "NVPTXISD::Tld4A2DS64Float";
  case NVPTXISD::Tld4R2DU64Float:
    return "NVPTXISD::Tld4R2DU64Float";
  case NVPTXISD::Tld4G2DU64Float:
    return "NVPTXISD::Tld4G2DU64Float";
  case NVPTXISD::Tld4B2DU64Float:
    return "NVPTXISD::Tld4B2DU64Float";
  case NVPTXISD::Tld4A2DU64Float:
    return "NVPTXISD::Tld4A2DU64Float";
 
  case NVPTXISD::TexUnified1DFloatS32:
    return "NVPTXISD::TexUnified1DFloatS32";
  case NVPTXISD::TexUnified1DFloatFloat:
    return "NVPTXISD::TexUnified1DFloatFloat";
  case NVPTXISD::TexUnified1DFloatFloatLevel:
    return "NVPTXISD::TexUnified1DFloatFloatLevel";
  case NVPTXISD::TexUnified1DFloatFloatGrad:
    return "NVPTXISD::TexUnified1DFloatFloatGrad";
  case NVPTXISD::TexUnified1DS32S32:
    return "NVPTXISD::TexUnified1DS32S32";
  case NVPTXISD::TexUnified1DS32Float:
    return "NVPTXISD::TexUnified1DS32Float";
  case NVPTXISD::TexUnified1DS32FloatLevel:
    return "NVPTXISD::TexUnified1DS32FloatLevel";
  case NVPTXISD::TexUnified1DS32FloatGrad:
    return "NVPTXISD::TexUnified1DS32FloatGrad";
  case NVPTXISD::TexUnified1DU32S32:
    return "NVPTXISD::TexUnified1DU32S32";
  case NVPTXISD::TexUnified1DU32Float:
    return "NVPTXISD::TexUnified1DU32Float";
  case NVPTXISD::TexUnified1DU32FloatLevel:
    return "NVPTXISD::TexUnified1DU32FloatLevel";
  case NVPTXISD::TexUnified1DU32FloatGrad:
    return "NVPTXISD::TexUnified1DU32FloatGrad";
  case NVPTXISD::TexUnified1DArrayFloatS32:
    return "NVPTXISD::TexUnified1DArrayFloatS32";
  case NVPTXISD::TexUnified1DArrayFloatFloat:
    return "NVPTXISD::TexUnified1DArrayFloatFloat";
  case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
    return "NVPTXISD::TexUnified1DArrayFloatFloatLevel";
  case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
    return "NVPTXISD::TexUnified1DArrayFloatFloatGrad";
  case NVPTXISD::TexUnified1DArrayS32S32:
    return "NVPTXISD::TexUnified1DArrayS32S32";
  case NVPTXISD::TexUnified1DArrayS32Float:
    return "NVPTXISD::TexUnified1DArrayS32Float";
  case NVPTXISD::TexUnified1DArrayS32FloatLevel:
    return "NVPTXISD::TexUnified1DArrayS32FloatLevel";
  case NVPTXISD::TexUnified1DArrayS32FloatGrad:
    return "NVPTXISD::TexUnified1DArrayS32FloatGrad";
  case NVPTXISD::TexUnified1DArrayU32S32:
    return "NVPTXISD::TexUnified1DArrayU32S32";
  case NVPTXISD::TexUnified1DArrayU32Float:
    return "NVPTXISD::TexUnified1DArrayU32Float";
  case NVPTXISD::TexUnified1DArrayU32FloatLevel:
    return "NVPTXISD::TexUnified1DArrayU32FloatLevel";
  case NVPTXISD::TexUnified1DArrayU32FloatGrad:
    return "NVPTXISD::TexUnified1DArrayU32FloatGrad";
  case NVPTXISD::TexUnified2DFloatS32:
    return "NVPTXISD::TexUnified2DFloatS32";
  case NVPTXISD::TexUnified2DFloatFloat:
    return "NVPTXISD::TexUnified2DFloatFloat";
  case NVPTXISD::TexUnified2DFloatFloatLevel:
    return "NVPTXISD::TexUnified2DFloatFloatLevel";
  case NVPTXISD::TexUnified2DFloatFloatGrad:
    return "NVPTXISD::TexUnified2DFloatFloatGrad";
  case NVPTXISD::TexUnified2DS32S32:
    return "NVPTXISD::TexUnified2DS32S32";
  case NVPTXISD::TexUnified2DS32Float:
    return "NVPTXISD::TexUnified2DS32Float";
  case NVPTXISD::TexUnified2DS32FloatLevel:
    return "NVPTXISD::TexUnified2DS32FloatLevel";
  case NVPTXISD::TexUnified2DS32FloatGrad:
    return "NVPTXISD::TexUnified2DS32FloatGrad";
  case NVPTXISD::TexUnified2DU32S32:
    return "NVPTXISD::TexUnified2DU32S32";
  case NVPTXISD::TexUnified2DU32Float:
    return "NVPTXISD::TexUnified2DU32Float";
  case NVPTXISD::TexUnified2DU32FloatLevel:
    return "NVPTXISD::TexUnified2DU32FloatLevel";
  case NVPTXISD::TexUnified2DU32FloatGrad:
    return "NVPTXISD::TexUnified2DU32FloatGrad";
  case NVPTXISD::TexUnified2DArrayFloatS32:
    return "NVPTXISD::TexUnified2DArrayFloatS32";
  case NVPTXISD::TexUnified2DArrayFloatFloat:
    return "NVPTXISD::TexUnified2DArrayFloatFloat";
  case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
    return "NVPTXISD::TexUnified2DArrayFloatFloatLevel";
  case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
    return "NVPTXISD::TexUnified2DArrayFloatFloatGrad";
  case NVPTXISD::TexUnified2DArrayS32S32:
    return "NVPTXISD::TexUnified2DArrayS32S32";
  case NVPTXISD::TexUnified2DArrayS32Float:
    return "NVPTXISD::TexUnified2DArrayS32Float";
  case NVPTXISD::TexUnified2DArrayS32FloatLevel:
    return "NVPTXISD::TexUnified2DArrayS32FloatLevel";
  case NVPTXISD::TexUnified2DArrayS32FloatGrad:
    return "NVPTXISD::TexUnified2DArrayS32FloatGrad";
  case NVPTXISD::TexUnified2DArrayU32S32:
    return "NVPTXISD::TexUnified2DArrayU32S32";
  case NVPTXISD::TexUnified2DArrayU32Float:
    return "NVPTXISD::TexUnified2DArrayU32Float";
  case NVPTXISD::TexUnified2DArrayU32FloatLevel:
    return "NVPTXISD::TexUnified2DArrayU32FloatLevel";
  case NVPTXISD::TexUnified2DArrayU32FloatGrad:
    return "NVPTXISD::TexUnified2DArrayU32FloatGrad";
  case NVPTXISD::TexUnified3DFloatS32:
    return "NVPTXISD::TexUnified3DFloatS32";
  case NVPTXISD::TexUnified3DFloatFloat:
    return "NVPTXISD::TexUnified3DFloatFloat";
  case NVPTXISD::TexUnified3DFloatFloatLevel:
    return "NVPTXISD::TexUnified3DFloatFloatLevel";
  case NVPTXISD::TexUnified3DFloatFloatGrad:
    return "NVPTXISD::TexUnified3DFloatFloatGrad";
  case NVPTXISD::TexUnified3DS32S32:
    return "NVPTXISD::TexUnified3DS32S32";
  case NVPTXISD::TexUnified3DS32Float:
    return "NVPTXISD::TexUnified3DS32Float";
  case NVPTXISD::TexUnified3DS32FloatLevel:
    return "NVPTXISD::TexUnified3DS32FloatLevel";
  case NVPTXISD::TexUnified3DS32FloatGrad:
    return "NVPTXISD::TexUnified3DS32FloatGrad";
  case NVPTXISD::TexUnified3DU32S32:
    return "NVPTXISD::TexUnified3DU32S32";
  case NVPTXISD::TexUnified3DU32Float:
    return "NVPTXISD::TexUnified3DU32Float";
  case NVPTXISD::TexUnified3DU32FloatLevel:
    return "NVPTXISD::TexUnified3DU32FloatLevel";
  case NVPTXISD::TexUnified3DU32FloatGrad:
    return "NVPTXISD::TexUnified3DU32FloatGrad";
  case NVPTXISD::TexUnifiedCubeFloatFloat:
    return "NVPTXISD::TexUnifiedCubeFloatFloat";
  case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
    return "NVPTXISD::TexUnifiedCubeFloatFloatLevel";
  case NVPTXISD::TexUnifiedCubeS32Float:
    return "NVPTXISD::TexUnifiedCubeS32Float";
  case NVPTXISD::TexUnifiedCubeS32FloatLevel:
    return "NVPTXISD::TexUnifiedCubeS32FloatLevel";
  case NVPTXISD::TexUnifiedCubeU32Float:
    return "NVPTXISD::TexUnifiedCubeU32Float";
  case NVPTXISD::TexUnifiedCubeU32FloatLevel:
    return "NVPTXISD::TexUnifiedCubeU32FloatLevel";
  case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
    return "NVPTXISD::TexUnifiedCubeArrayFloatFloat";
  case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
    return "NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel";
  case NVPTXISD::TexUnifiedCubeArrayS32Float:
    return "NVPTXISD::TexUnifiedCubeArrayS32Float";
  case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
    return "NVPTXISD::TexUnifiedCubeArrayS32FloatLevel";
  case NVPTXISD::TexUnifiedCubeArrayU32Float:
    return "NVPTXISD::TexUnifiedCubeArrayU32Float";
  case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
    return "NVPTXISD::TexUnifiedCubeArrayU32FloatLevel";
  case NVPTXISD::Tld4UnifiedR2DFloatFloat:
    return "NVPTXISD::Tld4UnifiedR2DFloatFloat";
  case NVPTXISD::Tld4UnifiedG2DFloatFloat:
    return "NVPTXISD::Tld4UnifiedG2DFloatFloat";
  case NVPTXISD::Tld4UnifiedB2DFloatFloat:
    return "NVPTXISD::Tld4UnifiedB2DFloatFloat";
  case NVPTXISD::Tld4UnifiedA2DFloatFloat:
    return "NVPTXISD::Tld4UnifiedA2DFloatFloat";
  case NVPTXISD::Tld4UnifiedR2DS64Float:
    return "NVPTXISD::Tld4UnifiedR2DS64Float";
  case NVPTXISD::Tld4UnifiedG2DS64Float:
    return "NVPTXISD::Tld4UnifiedG2DS64Float";
  case NVPTXISD::Tld4UnifiedB2DS64Float:
    return "NVPTXISD::Tld4UnifiedB2DS64Float";
  case NVPTXISD::Tld4UnifiedA2DS64Float:
    return "NVPTXISD::Tld4UnifiedA2DS64Float";
  case NVPTXISD::Tld4UnifiedR2DU64Float:
    return "NVPTXISD::Tld4UnifiedR2DU64Float";
  case NVPTXISD::Tld4UnifiedG2DU64Float:
    return "NVPTXISD::Tld4UnifiedG2DU64Float";
  case NVPTXISD::Tld4UnifiedB2DU64Float:
    return "NVPTXISD::Tld4UnifiedB2DU64Float";
  case NVPTXISD::Tld4UnifiedA2DU64Float:
    return "NVPTXISD::Tld4UnifiedA2DU64Float";
 
  case NVPTXISD::Suld1DI8Clamp:          return "NVPTXISD::Suld1DI8Clamp";
  case NVPTXISD::Suld1DI16Clamp:         return "NVPTXISD::Suld1DI16Clamp";
  case NVPTXISD::Suld1DI32Clamp:         return "NVPTXISD::Suld1DI32Clamp";
  case NVPTXISD::Suld1DI64Clamp:         return "NVPTXISD::Suld1DI64Clamp";
  case NVPTXISD::Suld1DV2I8Clamp:        return "NVPTXISD::Suld1DV2I8Clamp";
  case NVPTXISD::Suld1DV2I16Clamp:       return "NVPTXISD::Suld1DV2I16Clamp";
  case NVPTXISD::Suld1DV2I32Clamp:       return "NVPTXISD::Suld1DV2I32Clamp";
  case NVPTXISD::Suld1DV2I64Clamp:       return "NVPTXISD::Suld1DV2I64Clamp";
  case NVPTXISD::Suld1DV4I8Clamp:        return "NVPTXISD::Suld1DV4I8Clamp";
  case NVPTXISD::Suld1DV4I16Clamp:       return "NVPTXISD::Suld1DV4I16Clamp";
  case NVPTXISD::Suld1DV4I32Clamp:       return "NVPTXISD::Suld1DV4I32Clamp";
 
  case NVPTXISD::Suld1DArrayI8Clamp:   return "NVPTXISD::Suld1DArrayI8Clamp";
  case NVPTXISD::Suld1DArrayI16Clamp:  return "NVPTXISD::Suld1DArrayI16Clamp";
  case NVPTXISD::Suld1DArrayI32Clamp:  return "NVPTXISD::Suld1DArrayI32Clamp";
  case NVPTXISD::Suld1DArrayI64Clamp:  return "NVPTXISD::Suld1DArrayI64Clamp";
  case NVPTXISD::Suld1DArrayV2I8Clamp: return "NVPTXISD::Suld1DArrayV2I8Clamp";
  case NVPTXISD::Suld1DArrayV2I16Clamp:return "NVPTXISD::Suld1DArrayV2I16Clamp";
  case NVPTXISD::Suld1DArrayV2I32Clamp:return "NVPTXISD::Suld1DArrayV2I32Clamp";
  case NVPTXISD::Suld1DArrayV2I64Clamp:return "NVPTXISD::Suld1DArrayV2I64Clamp";
  case NVPTXISD::Suld1DArrayV4I8Clamp: return "NVPTXISD::Suld1DArrayV4I8Clamp";
  case NVPTXISD::Suld1DArrayV4I16Clamp:return "NVPTXISD::Suld1DArrayV4I16Clamp";
  case NVPTXISD::Suld1DArrayV4I32Clamp:return "NVPTXISD::Suld1DArrayV4I32Clamp";
 
  case NVPTXISD::Suld2DI8Clamp:          return "NVPTXISD::Suld2DI8Clamp";
  case NVPTXISD::Suld2DI16Clamp:         return "NVPTXISD::Suld2DI16Clamp";
  case NVPTXISD::Suld2DI32Clamp:         return "NVPTXISD::Suld2DI32Clamp";
  case NVPTXISD::Suld2DI64Clamp:         return "NVPTXISD::Suld2DI64Clamp";
  case NVPTXISD::Suld2DV2I8Clamp:        return "NVPTXISD::Suld2DV2I8Clamp";
  case NVPTXISD::Suld2DV2I16Clamp:       return "NVPTXISD::Suld2DV2I16Clamp";
  case NVPTXISD::Suld2DV2I32Clamp:       return "NVPTXISD::Suld2DV2I32Clamp";
  case NVPTXISD::Suld2DV2I64Clamp:       return "NVPTXISD::Suld2DV2I64Clamp";
  case NVPTXISD::Suld2DV4I8Clamp:        return "NVPTXISD::Suld2DV4I8Clamp";
  case NVPTXISD::Suld2DV4I16Clamp:       return "NVPTXISD::Suld2DV4I16Clamp";
  case NVPTXISD::Suld2DV4I32Clamp:       return "NVPTXISD::Suld2DV4I32Clamp";
 
  case NVPTXISD::Suld2DArrayI8Clamp:   return "NVPTXISD::Suld2DArrayI8Clamp";
  case NVPTXISD::Suld2DArrayI16Clamp:  return "NVPTXISD::Suld2DArrayI16Clamp";
  case NVPTXISD::Suld2DArrayI32Clamp:  return "NVPTXISD::Suld2DArrayI32Clamp";
  case NVPTXISD::Suld2DArrayI64Clamp:  return "NVPTXISD::Suld2DArrayI64Clamp";
  case NVPTXISD::Suld2DArrayV2I8Clamp: return "NVPTXISD::Suld2DArrayV2I8Clamp";
  case NVPTXISD::Suld2DArrayV2I16Clamp:return "NVPTXISD::Suld2DArrayV2I16Clamp";
  case NVPTXISD::Suld2DArrayV2I32Clamp:return "NVPTXISD::Suld2DArrayV2I32Clamp";
  case NVPTXISD::Suld2DArrayV2I64Clamp:return "NVPTXISD::Suld2DArrayV2I64Clamp";
  case NVPTXISD::Suld2DArrayV4I8Clamp: return "NVPTXISD::Suld2DArrayV4I8Clamp";
  case NVPTXISD::Suld2DArrayV4I16Clamp:return "NVPTXISD::Suld2DArrayV4I16Clamp";
  case NVPTXISD::Suld2DArrayV4I32Clamp:return "NVPTXISD::Suld2DArrayV4I32Clamp";
 
  case NVPTXISD::Suld3DI8Clamp:          return "NVPTXISD::Suld3DI8Clamp";
  case NVPTXISD::Suld3DI16Clamp:         return "NVPTXISD::Suld3DI16Clamp";
  case NVPTXISD::Suld3DI32Clamp:         return "NVPTXISD::Suld3DI32Clamp";
  case NVPTXISD::Suld3DI64Clamp:         return "NVPTXISD::Suld3DI64Clamp";
  case NVPTXISD::Suld3DV2I8Clamp:        return "NVPTXISD::Suld3DV2I8Clamp";
  case NVPTXISD::Suld3DV2I16Clamp:       return "NVPTXISD::Suld3DV2I16Clamp";
  case NVPTXISD::Suld3DV2I32Clamp:       return "NVPTXISD::Suld3DV2I32Clamp";
  case NVPTXISD::Suld3DV2I64Clamp:       return "NVPTXISD::Suld3DV2I64Clamp";
  case NVPTXISD::Suld3DV4I8Clamp:        return "NVPTXISD::Suld3DV4I8Clamp";
  case NVPTXISD::Suld3DV4I16Clamp:       return "NVPTXISD::Suld3DV4I16Clamp";
  case NVPTXISD::Suld3DV4I32Clamp:       return "NVPTXISD::Suld3DV4I32Clamp";
 
  case NVPTXISD::Suld1DI8Trap:          return "NVPTXISD::Suld1DI8Trap";
  case NVPTXISD::Suld1DI16Trap:         return "NVPTXISD::Suld1DI16Trap";
  case NVPTXISD::Suld1DI32Trap:         return "NVPTXISD::Suld1DI32Trap";
  case NVPTXISD::Suld1DI64Trap:         return "NVPTXISD::Suld1DI64Trap";
  case NVPTXISD::Suld1DV2I8Trap:        return "NVPTXISD::Suld1DV2I8Trap";
  case NVPTXISD::Suld1DV2I16Trap:       return "NVPTXISD::Suld1DV2I16Trap";
  case NVPTXISD::Suld1DV2I32Trap:       return "NVPTXISD::Suld1DV2I32Trap";
  case NVPTXISD::Suld1DV2I64Trap:       return "NVPTXISD::Suld1DV2I64Trap";
  case NVPTXISD::Suld1DV4I8Trap:        return "NVPTXISD::Suld1DV4I8Trap";
  case NVPTXISD::Suld1DV4I16Trap:       return "NVPTXISD::Suld1DV4I16Trap";
  case NVPTXISD::Suld1DV4I32Trap:       return "NVPTXISD::Suld1DV4I32Trap";
 
  case NVPTXISD::Suld1DArrayI8Trap:     return "NVPTXISD::Suld1DArrayI8Trap";
  case NVPTXISD::Suld1DArrayI16Trap:    return "NVPTXISD::Suld1DArrayI16Trap";
  case NVPTXISD::Suld1DArrayI32Trap:    return "NVPTXISD::Suld1DArrayI32Trap";
  case NVPTXISD::Suld1DArrayI64Trap:    return "NVPTXISD::Suld1DArrayI64Trap";
  case NVPTXISD::Suld1DArrayV2I8Trap:   return "NVPTXISD::Suld1DArrayV2I8Trap";
  case NVPTXISD::Suld1DArrayV2I16Trap:  return "NVPTXISD::Suld1DArrayV2I16Trap";
  case NVPTXISD::Suld1DArrayV2I32Trap:  return "NVPTXISD::Suld1DArrayV2I32Trap";
  case NVPTXISD::Suld1DArrayV2I64Trap:  return "NVPTXISD::Suld1DArrayV2I64Trap";
  case NVPTXISD::Suld1DArrayV4I8Trap:   return "NVPTXISD::Suld1DArrayV4I8Trap";
  case NVPTXISD::Suld1DArrayV4I16Trap:  return "NVPTXISD::Suld1DArrayV4I16Trap";
  case NVPTXISD::Suld1DArrayV4I32Trap:  return "NVPTXISD::Suld1DArrayV4I32Trap";
 
  case NVPTXISD::Suld2DI8Trap:          return "NVPTXISD::Suld2DI8Trap";
  case NVPTXISD::Suld2DI16Trap:         return "NVPTXISD::Suld2DI16Trap";
  case NVPTXISD::Suld2DI32Trap:         return "NVPTXISD::Suld2DI32Trap";
  case NVPTXISD::Suld2DI64Trap:         return "NVPTXISD::Suld2DI64Trap";
  case NVPTXISD::Suld2DV2I8Trap:        return "NVPTXISD::Suld2DV2I8Trap";
  case NVPTXISD::Suld2DV2I16Trap:       return "NVPTXISD::Suld2DV2I16Trap";
  case NVPTXISD::Suld2DV2I32Trap:       return "NVPTXISD::Suld2DV2I32Trap";
  case NVPTXISD::Suld2DV2I64Trap:       return "NVPTXISD::Suld2DV2I64Trap";
  case NVPTXISD::Suld2DV4I8Trap:        return "NVPTXISD::Suld2DV4I8Trap";
  case NVPTXISD::Suld2DV4I16Trap:       return "NVPTXISD::Suld2DV4I16Trap";
  case NVPTXISD::Suld2DV4I32Trap:       return "NVPTXISD::Suld2DV4I32Trap";
 
  case NVPTXISD::Suld2DArrayI8Trap:     return "NVPTXISD::Suld2DArrayI8Trap";
  case NVPTXISD::Suld2DArrayI16Trap:    return "NVPTXISD::Suld2DArrayI16Trap";
  case NVPTXISD::Suld2DArrayI32Trap:    return "NVPTXISD::Suld2DArrayI32Trap";
  case NVPTXISD::Suld2DArrayI64Trap:    return "NVPTXISD::Suld2DArrayI64Trap";
  case NVPTXISD::Suld2DArrayV2I8Trap:   return "NVPTXISD::Suld2DArrayV2I8Trap";
  case NVPTXISD::Suld2DArrayV2I16Trap:  return "NVPTXISD::Suld2DArrayV2I16Trap";
  case NVPTXISD::Suld2DArrayV2I32Trap:  return "NVPTXISD::Suld2DArrayV2I32Trap";
  case NVPTXISD::Suld2DArrayV2I64Trap:  return "NVPTXISD::Suld2DArrayV2I64Trap";
  case NVPTXISD::Suld2DArrayV4I8Trap:   return "NVPTXISD::Suld2DArrayV4I8Trap";
  case NVPTXISD::Suld2DArrayV4I16Trap:  return "NVPTXISD::Suld2DArrayV4I16Trap";
  case NVPTXISD::Suld2DArrayV4I32Trap:  return "NVPTXISD::Suld2DArrayV4I32Trap";
 
  case NVPTXISD::Suld3DI8Trap:          return "NVPTXISD::Suld3DI8Trap";
  case NVPTXISD::Suld3DI16Trap:         return "NVPTXISD::Suld3DI16Trap";
  case NVPTXISD::Suld3DI32Trap:         return "NVPTXISD::Suld3DI32Trap";
  case NVPTXISD::Suld3DI64Trap:         return "NVPTXISD::Suld3DI64Trap";
  case NVPTXISD::Suld3DV2I8Trap:        return "NVPTXISD::Suld3DV2I8Trap";
  case NVPTXISD::Suld3DV2I16Trap:       return "NVPTXISD::Suld3DV2I16Trap";
  case NVPTXISD::Suld3DV2I32Trap:       return "NVPTXISD::Suld3DV2I32Trap";
  case NVPTXISD::Suld3DV2I64Trap:       return "NVPTXISD::Suld3DV2I64Trap";
  case NVPTXISD::Suld3DV4I8Trap:        return "NVPTXISD::Suld3DV4I8Trap";
  case NVPTXISD::Suld3DV4I16Trap:       return "NVPTXISD::Suld3DV4I16Trap";
  case NVPTXISD::Suld3DV4I32Trap:       return "NVPTXISD::Suld3DV4I32Trap";
 
  case NVPTXISD::Suld1DI8Zero:          return "NVPTXISD::Suld1DI8Zero";
  case NVPTXISD::Suld1DI16Zero:         return "NVPTXISD::Suld1DI16Zero";
  case NVPTXISD::Suld1DI32Zero:         return "NVPTXISD::Suld1DI32Zero";
  case NVPTXISD::Suld1DI64Zero:         return "NVPTXISD::Suld1DI64Zero";
  case NVPTXISD::Suld1DV2I8Zero:        return "NVPTXISD::Suld1DV2I8Zero";
  case NVPTXISD::Suld1DV2I16Zero:       return "NVPTXISD::Suld1DV2I16Zero";
  case NVPTXISD::Suld1DV2I32Zero:       return "NVPTXISD::Suld1DV2I32Zero";
  case NVPTXISD::Suld1DV2I64Zero:       return "NVPTXISD::Suld1DV2I64Zero";
  case NVPTXISD::Suld1DV4I8Zero:        return "NVPTXISD::Suld1DV4I8Zero";
  case NVPTXISD::Suld1DV4I16Zero:       return "NVPTXISD::Suld1DV4I16Zero";
  case NVPTXISD::Suld1DV4I32Zero:       return "NVPTXISD::Suld1DV4I32Zero";
 
  case NVPTXISD::Suld1DArrayI8Zero:     return "NVPTXISD::Suld1DArrayI8Zero";
  case NVPTXISD::Suld1DArrayI16Zero:    return "NVPTXISD::Suld1DArrayI16Zero";
  case NVPTXISD::Suld1DArrayI32Zero:    return "NVPTXISD::Suld1DArrayI32Zero";
  case NVPTXISD::Suld1DArrayI64Zero:    return "NVPTXISD::Suld1DArrayI64Zero";
  case NVPTXISD::Suld1DArrayV2I8Zero:   return "NVPTXISD::Suld1DArrayV2I8Zero";
  case NVPTXISD::Suld1DArrayV2I16Zero:  return "NVPTXISD::Suld1DArrayV2I16Zero";
  case NVPTXISD::Suld1DArrayV2I32Zero:  return "NVPTXISD::Suld1DArrayV2I32Zero";
  case NVPTXISD::Suld1DArrayV2I64Zero:  return "NVPTXISD::Suld1DArrayV2I64Zero";
  case NVPTXISD::Suld1DArrayV4I8Zero:   return "NVPTXISD::Suld1DArrayV4I8Zero";
  case NVPTXISD::Suld1DArrayV4I16Zero:  return "NVPTXISD::Suld1DArrayV4I16Zero";
  case NVPTXISD::Suld1DArrayV4I32Zero:  return "NVPTXISD::Suld1DArrayV4I32Zero";
 
  case NVPTXISD::Suld2DI8Zero:          return "NVPTXISD::Suld2DI8Zero";
  case NVPTXISD::Suld2DI16Zero:         return "NVPTXISD::Suld2DI16Zero";
  case NVPTXISD::Suld2DI32Zero:         return "NVPTXISD::Suld2DI32Zero";
  case NVPTXISD::Suld2DI64Zero:         return "NVPTXISD::Suld2DI64Zero";
  case NVPTXISD::Suld2DV2I8Zero:        return "NVPTXISD::Suld2DV2I8Zero";
  case NVPTXISD::Suld2DV2I16Zero:       return "NVPTXISD::Suld2DV2I16Zero";
  case NVPTXISD::Suld2DV2I32Zero:       return "NVPTXISD::Suld2DV2I32Zero";
  case NVPTXISD::Suld2DV2I64Zero:       return "NVPTXISD::Suld2DV2I64Zero";
  case NVPTXISD::Suld2DV4I8Zero:        return "NVPTXISD::Suld2DV4I8Zero";
  case NVPTXISD::Suld2DV4I16Zero:       return "NVPTXISD::Suld2DV4I16Zero";
  case NVPTXISD::Suld2DV4I32Zero:       return "NVPTXISD::Suld2DV4I32Zero";
 
  case NVPTXISD::Suld2DArrayI8Zero:     return "NVPTXISD::Suld2DArrayI8Zero";
  case NVPTXISD::Suld2DArrayI16Zero:    return "NVPTXISD::Suld2DArrayI16Zero";
  case NVPTXISD::Suld2DArrayI32Zero:    return "NVPTXISD::Suld2DArrayI32Zero";
  case NVPTXISD::Suld2DArrayI64Zero:    return "NVPTXISD::Suld2DArrayI64Zero";
  case NVPTXISD::Suld2DArrayV2I8Zero:   return "NVPTXISD::Suld2DArrayV2I8Zero";
  case NVPTXISD::Suld2DArrayV2I16Zero:  return "NVPTXISD::Suld2DArrayV2I16Zero";
  case NVPTXISD::Suld2DArrayV2I32Zero:  return "NVPTXISD::Suld2DArrayV2I32Zero";
  case NVPTXISD::Suld2DArrayV2I64Zero:  return "NVPTXISD::Suld2DArrayV2I64Zero";
  case NVPTXISD::Suld2DArrayV4I8Zero:   return "NVPTXISD::Suld2DArrayV4I8Zero";
  case NVPTXISD::Suld2DArrayV4I16Zero:  return "NVPTXISD::Suld2DArrayV4I16Zero";
  case NVPTXISD::Suld2DArrayV4I32Zero:  return "NVPTXISD::Suld2DArrayV4I32Zero";
 
  case NVPTXISD::Suld3DI8Zero:          return "NVPTXISD::Suld3DI8Zero";
  case NVPTXISD::Suld3DI16Zero:         return "NVPTXISD::Suld3DI16Zero";
  case NVPTXISD::Suld3DI32Zero:         return "NVPTXISD::Suld3DI32Zero";
  case NVPTXISD::Suld3DI64Zero:         return "NVPTXISD::Suld3DI64Zero";
  case NVPTXISD::Suld3DV2I8Zero:        return "NVPTXISD::Suld3DV2I8Zero";
  case NVPTXISD::Suld3DV2I16Zero:       return "NVPTXISD::Suld3DV2I16Zero";
  case NVPTXISD::Suld3DV2I32Zero:       return "NVPTXISD::Suld3DV2I32Zero";
  case NVPTXISD::Suld3DV2I64Zero:       return "NVPTXISD::Suld3DV2I64Zero";
  case NVPTXISD::Suld3DV4I8Zero:        return "NVPTXISD::Suld3DV4I8Zero";
  case NVPTXISD::Suld3DV4I16Zero:       return "NVPTXISD::Suld3DV4I16Zero";
  case NVPTXISD::Suld3DV4I32Zero:       return "NVPTXISD::Suld3DV4I32Zero";
  }
  return nullptr;
}
 
TargetLoweringBase::LegalizeTypeAction
NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {
  if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
      VT.getScalarType() == MVT::i1)
    return TypeSplitVector;
  if (Isv2x16VT(VT))
    return TypeLegal;
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
                                             int Enabled, int &ExtraSteps,
                                             bool &UseOneConst,
                                             bool Reciprocal) const {
  if (!(Enabled == ReciprocalEstimate::Enabled ||
        (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
    return SDValue();
 
  if (ExtraSteps == ReciprocalEstimate::Unspecified)
    ExtraSteps = 0;
 
  SDLoc DL(Operand);
  EVT VT = Operand.getValueType();
  bool Ftz = useF32FTZ(DAG.getMachineFunction());
 
  auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {
    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
                       DAG.getConstant(IID, DL, MVT::i32), Operand);
  };
 
  // The sqrt and rsqrt refinement processes assume we always start out with an
  // approximation of the rsqrt.  Therefore, if we're going to do any refinement
  // (i.e. ExtraSteps > 0), we must return an rsqrt.  But if we're *not* doing
  // any refinement, we must return a regular sqrt.
  if (Reciprocal || ExtraSteps > 0) {
    if (VT == MVT::f32)
      return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
                                   : Intrinsic::nvvm_rsqrt_approx_f);
    else if (VT == MVT::f64)
      return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
    else
      return SDValue();
  } else {
    if (VT == MVT::f32)
      return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
                                   : Intrinsic::nvvm_sqrt_approx_f);
    else {
      // There's no sqrt.approx.f64 instruction, so we emit
      // reciprocal(rsqrt(x)).  This is faster than
      // select(x == 0, 0, x * rsqrt(x)).  (In fact, it's faster than plain
      // x * rsqrt(x).)
      return DAG.getNode(
          ISD::INTRINSIC_WO_CHAIN, DL, VT,
          DAG.getConstant(Intrinsic::nvvm_rcp_approx_ftz_d, DL, MVT::i32),
          MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
    }
  }
}
 
SDValue
NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
  SDLoc dl(Op);
  const GlobalAddressSDNode *GAN = cast<GlobalAddressSDNode>(Op);
  auto PtrVT = getPointerTy(DAG.getDataLayout(), GAN->getAddressSpace());
  Op = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, PtrVT);
  return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op);
}
 
static bool IsTypePassedAsArray(const Type *Ty) {
  return Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128) ||
         Ty->isHalfTy() || Ty->isBFloatTy();
}
 
std::string NVPTXTargetLowering::getPrototype(
    const DataLayout &DL, Type *retTy, const ArgListTy &Args,
    const SmallVectorImpl<ISD::OutputArg> &Outs, MaybeAlign retAlignment,
    std::optional<std::pair<unsigned, const APInt &>> VAInfo,
    const CallBase &CB, unsigned UniqueCallSite) const {
  auto PtrVT = getPointerTy(DL);
 
  bool isABI = (STI.getSmVersion() >= 20);
  assert(isABI && "Non-ABI compilation is not supported");
  if (!isABI)
    return "";
 
  std::string Prototype;
  raw_string_ostream O(Prototype);
  O << "prototype_" << UniqueCallSite << " : .callprototype ";
 
  if (retTy->getTypeID() == Type::VoidTyID) {
    O << "()";
  } else {
    O << "(";
    if ((retTy->isFloatingPointTy() || retTy->isIntegerTy()) &&
        !IsTypePassedAsArray(retTy)) {
      unsigned size = 0;
      if (auto *ITy = dyn_cast<IntegerType>(retTy)) {
        size = ITy->getBitWidth();
      } else {
        assert(retTy->isFloatingPointTy() &&
               "Floating point type expected here");
        size = retTy->getPrimitiveSizeInBits();
      }
      // PTX ABI requires all scalar return values to be at least 32
      // bits in size.  fp16 normally uses .b16 as its storage type in
      // PTX, so its size must be adjusted here, too.
      size = promoteScalarArgumentSize(size);
 
      O << ".param .b" << size << " _";
    } else if (isa<PointerType>(retTy)) {
      O << ".param .b" << PtrVT.getSizeInBits() << " _";
    } else if (IsTypePassedAsArray(retTy)) {
      O << ".param .align " << (retAlignment ? retAlignment->value() : 0)
        << " .b8 _[" << DL.getTypeAllocSize(retTy) << "]";
    } else {
      llvm_unreachable("Unknown return type");
    }
    O << ") ";
  }
  O << "_ (";
 
  bool first = true;
 
  const Function *F = CB.getFunction();
  unsigned NumArgs = VAInfo ? VAInfo->first : Args.size();
  for (unsigned i = 0, OIdx = 0; i != NumArgs; ++i, ++OIdx) {
    Type *Ty = Args[i].Ty;
    if (!first) {
      O << ", ";
    }
    first = false;
 
    if (!Outs[OIdx].Flags.isByVal()) {
      if (IsTypePassedAsArray(Ty)) {
        unsigned ParamAlign = 0;
        const CallInst *CallI = cast<CallInst>(&CB);
        // +1 because index 0 is reserved for return type alignment
        if (!getAlign(*CallI, i + 1, ParamAlign))
          ParamAlign = getFunctionParamOptimizedAlign(F, Ty, DL).value();
        O << ".param .align " << ParamAlign << " .b8 ";
        O << "_";
        O << "[" << DL.getTypeAllocSize(Ty) << "]";
        // update the index for Outs
        SmallVector<EVT, 16> vtparts;
        ComputeValueVTs(*this, DL, Ty, vtparts);
        if (unsigned len = vtparts.size())
          OIdx += len - 1;
        continue;
      }
      // i8 types in IR will be i16 types in SDAG
      assert((getValueType(DL, Ty) == Outs[OIdx].VT ||
              (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
             "type mismatch between callee prototype and arguments");
      // scalar type
      unsigned sz = 0;
      if (isa<IntegerType>(Ty)) {
        sz = cast<IntegerType>(Ty)->getBitWidth();
        sz = promoteScalarArgumentSize(sz);
      } else if (isa<PointerType>(Ty)) {
        sz = PtrVT.getSizeInBits();
      } else {
        sz = Ty->getPrimitiveSizeInBits();
      }
      O << ".param .b" << sz << " ";
      O << "_";
      continue;
    }
 
    Type *ETy = Args[i].IndirectType;
    Align InitialAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
    Align ParamByValAlign =
        getFunctionByValParamAlign(F, ETy, InitialAlign, DL);
 
    O << ".param .align " << ParamByValAlign.value() << " .b8 ";
    O << "_";
    O << "[" << Outs[OIdx].Flags.getByValSize() << "]";
  }
 
  if (VAInfo)
    O << (first ? "" : ",") << " .param .align " << VAInfo->second
      << " .b8 _[]\n";
  O << ")";
  if (shouldEmitPTXNoReturn(&CB, *nvTM))
    O << " .noreturn";
  O << ";";
 
  return Prototype;
}
 
Align NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
                                                const CallBase *CB, Type *Ty,
                                                unsigned Idx,
                                                const DataLayout &DL) const {
  if (!CB) {
    // CallSite is zero, fallback to ABI type alignment
    return DL.getABITypeAlign(Ty);
  }
 
  unsigned Alignment = 0;
  const Function *DirectCallee = CB->getCalledFunction();
 
  if (!DirectCallee) {
    // We don't have a direct function symbol, but that may be because of
    // constant cast instructions in the call.
 
    // With bitcast'd call targets, the instruction will be the call
    if (const auto *CI = dyn_cast<CallInst>(CB)) {
      // Check if we have call alignment metadata
      if (getAlign(*CI, Idx, Alignment))
        return Align(Alignment);
    }
    DirectCallee = getMaybeBitcastedCallee(CB);
  }
 
  // Check for function alignment information if we found that the
  // ultimate target is a Function
  if (DirectCallee) {
    if (getAlign(*DirectCallee, Idx, Alignment))
      return Align(Alignment);
    // If alignment information is not available, fall back to the
    // default function param optimized type alignment
    return getFunctionParamOptimizedAlign(DirectCallee, Ty, DL);
  }
 
  // Call is indirect, fall back to the ABI type alignment
  return DL.getABITypeAlign(Ty);
}
 
SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                       SmallVectorImpl<SDValue> &InVals) const {
 
  if (CLI.IsVarArg && (STI.getPTXVersion() < 60 || STI.getSmVersion() < 30))
    report_fatal_error(
        "Support for variadic functions (unsized array parameter) introduced "
        "in PTX ISA version 6.0 and requires target sm_30.");
 
  SelectionDAG &DAG = CLI.DAG;
  SDLoc dl = CLI.DL;
  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
  SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
  SDValue Chain = CLI.Chain;
  SDValue Callee = CLI.Callee;
  bool &isTailCall = CLI.IsTailCall;
  ArgListTy &Args = CLI.getArgs();
  Type *RetTy = CLI.RetTy;
  const CallBase *CB = CLI.CB;
  const DataLayout &DL = DAG.getDataLayout();
 
  bool isABI = (STI.getSmVersion() >= 20);
  assert(isABI && "Non-ABI compilation is not supported");
  if (!isABI)
    return Chain;
 
  // Variadic arguments.
  //
  // Normally, for each argument, we declare a param scalar or a param
  // byte array in the .param space, and store the argument value to that
  // param scalar or array starting at offset 0.
  //
  // In the case of the first variadic argument, we declare a vararg byte array
  // with size 0. The exact size of this array isn't known at this point, so
  // it'll be patched later. All the variadic arguments will be stored to this
  // array at a certain offset (which gets tracked by 'VAOffset'). The offset is
  // initially set to 0, so it can be used for non-variadic arguments (which use
  // 0 offset) to simplify the code.
  //
  // After all vararg is processed, 'VAOffset' holds the size of the
  // vararg byte array.
 
  SDValue VADeclareParam;                 // vararg byte array
  unsigned FirstVAArg = CLI.NumFixedArgs; // position of the first variadic
  unsigned VAOffset = 0;                  // current offset in the param array
 
  unsigned UniqueCallSite = GlobalUniqueCallSite.fetch_add(1);
  SDValue TempChain = Chain;
  Chain = DAG.getCALLSEQ_START(Chain, UniqueCallSite, 0, dl);
  SDValue InGlue = Chain.getValue(1);
 
  unsigned ParamCount = 0;
  // Args.size() and Outs.size() need not match.
  // Outs.size() will be larger
  //   * if there is an aggregate argument with multiple fields (each field
  //     showing up separately in Outs)
  //   * if there is a vector argument with more than typical vector-length
  //     elements (generally if more than 4) where each vector element is
  //     individually present in Outs.
  // So a different index should be used for indexing into Outs/OutVals.
  // See similar issue in LowerFormalArguments.
  unsigned OIdx = 0;
  // Declare the .params or .reg need to pass values
  // to the function
  for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
    EVT VT = Outs[OIdx].VT;
    Type *Ty = Args[i].Ty;
    bool IsVAArg = (i >= CLI.NumFixedArgs);
    bool IsByVal = Outs[OIdx].Flags.isByVal();
 
    SmallVector<EVT, 16> VTs;
    SmallVector<uint64_t, 16> Offsets;
 
    assert((!IsByVal || Args[i].IndirectType) &&
           "byval arg must have indirect type");
    Type *ETy = (IsByVal ? Args[i].IndirectType : Ty);
    ComputePTXValueVTs(*this, DL, ETy, VTs, &Offsets, IsByVal ? 0 : VAOffset);
 
    Align ArgAlign;
    if (IsByVal) {
      // The ByValAlign in the Outs[OIdx].Flags is always set at this point,
      // so we don't need to worry whether it's naturally aligned or not.
      // See TargetLowering::LowerCallTo().
      Align InitialAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
      ArgAlign = getFunctionByValParamAlign(CB->getCalledFunction(), ETy,
                                            InitialAlign, DL);
      if (IsVAArg)
        VAOffset = alignTo(VAOffset, ArgAlign);
    } else {
      ArgAlign = getArgumentAlignment(Callee, CB, Ty, ParamCount + 1, DL);
    }
 
    unsigned TypeSize =
        (IsByVal ? Outs[OIdx].Flags.getByValSize() : DL.getTypeAllocSize(Ty));
    SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
 
    bool NeedAlign; // Does argument declaration specify alignment?
    bool PassAsArray = IsByVal || IsTypePassedAsArray(Ty);
    if (IsVAArg) {
      if (ParamCount == FirstVAArg) {
        SDValue DeclareParamOps[] = {
            Chain, DAG.getConstant(STI.getMaxRequiredAlignment(), dl, MVT::i32),
            DAG.getConstant(ParamCount, dl, MVT::i32),
            DAG.getConstant(1, dl, MVT::i32), InGlue};
        VADeclareParam = Chain = DAG.getNode(NVPTXISD::DeclareParam, dl,
                                             DeclareParamVTs, DeclareParamOps);
      }
      NeedAlign = PassAsArray;
    } else if (PassAsArray) {
      // declare .param .align <align> .b8 .param<n>[<size>];
      SDValue DeclareParamOps[] = {
          Chain, DAG.getConstant(ArgAlign.value(), dl, MVT::i32),
          DAG.getConstant(ParamCount, dl, MVT::i32),
          DAG.getConstant(TypeSize, dl, MVT::i32), InGlue};
      Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
                          DeclareParamOps);
      NeedAlign = true;
    } else {
      // declare .param .b<size> .param<n>;
      if (VT.isInteger() || VT.isFloatingPoint()) {
        // PTX ABI requires integral types to be at least 32 bits in
        // size. FP16 is loaded/stored using i16, so it's handled
        // here as well.
        TypeSize = promoteScalarArgumentSize(TypeSize * 8) / 8;
      }
      SDValue DeclareScalarParamOps[] = {
          Chain, DAG.getConstant(ParamCount, dl, MVT::i32),
          DAG.getConstant(TypeSize * 8, dl, MVT::i32),
          DAG.getConstant(0, dl, MVT::i32), InGlue};
      Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,
                          DeclareScalarParamOps);
      NeedAlign = false;
    }
    InGlue = Chain.getValue(1);
 
    // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
    // than 32-bits are sign extended or zero extended, depending on
    // whether they are signed or unsigned types. This case applies
    // only to scalar parameters and not to aggregate values.
    bool ExtendIntegerParam =
        Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty) < 32;
 
    auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, ArgAlign, IsVAArg);
    SmallVector<SDValue, 6> StoreOperands;
    for (unsigned j = 0, je = VTs.size(); j != je; ++j) {
      EVT EltVT = VTs[j];
      int CurOffset = Offsets[j];
      MaybeAlign PartAlign;
      if (NeedAlign)
        PartAlign = commonAlignment(ArgAlign, CurOffset);
 
      // New store.
      if (VectorInfo[j] & PVF_FIRST) {
        assert(StoreOperands.empty() && "Unfinished preceding store.");
        StoreOperands.push_back(Chain);
        StoreOperands.push_back(
            DAG.getConstant(IsVAArg ? FirstVAArg : ParamCount, dl, MVT::i32));
        StoreOperands.push_back(DAG.getConstant(
            IsByVal ? CurOffset + VAOffset : (IsVAArg ? VAOffset : CurOffset),
            dl, MVT::i32));
      }
 
      SDValue StVal = OutVals[OIdx];
 
      MVT PromotedVT;
      if (PromoteScalarIntegerPTX(EltVT, &PromotedVT)) {
        EltVT = EVT(PromotedVT);
      }
      if (PromoteScalarIntegerPTX(StVal.getValueType(), &PromotedVT)) {
        llvm::ISD::NodeType Ext =
            Outs[OIdx].Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
        StVal = DAG.getNode(Ext, dl, PromotedVT, StVal);
      }
 
      if (IsByVal) {
        auto PtrVT = getPointerTy(DL);
        SDValue srcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StVal,
                                      DAG.getConstant(CurOffset, dl, PtrVT));
        StVal = DAG.getLoad(EltVT, dl, TempChain, srcAddr, MachinePointerInfo(),
                            PartAlign);
      } else if (ExtendIntegerParam) {
        assert(VTs.size() == 1 && "Scalar can't have multiple parts.");
        // zext/sext to i32
        StVal = DAG.getNode(Outs[OIdx].Flags.isSExt() ? ISD::SIGN_EXTEND
                                                      : ISD::ZERO_EXTEND,
                            dl, MVT::i32, StVal);
      }
 
      if (!ExtendIntegerParam && EltVT.getSizeInBits() < 16) {
        // Use 16-bit registers for small stores as it's the
        // smallest general purpose register size supported by NVPTX.
        StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal);
      }
 
      // Record the value to store.
      StoreOperands.push_back(StVal);
 
      if (VectorInfo[j] & PVF_LAST) {
        unsigned NumElts = StoreOperands.size() - 3;
        NVPTXISD::NodeType Op;
        switch (NumElts) {
        case 1:
          Op = NVPTXISD::StoreParam;
          break;
        case 2:
          Op = NVPTXISD::StoreParamV2;
          break;
        case 4:
          Op = NVPTXISD::StoreParamV4;
          break;
        default:
          llvm_unreachable("Invalid vector info.");
        }
 
        StoreOperands.push_back(InGlue);
 
        // Adjust type of the store op if we've extended the scalar
        // return value.
        EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
 
        Chain = DAG.getMemIntrinsicNode(
            Op, dl, DAG.getVTList(MVT::Other, MVT::Glue), StoreOperands,
            TheStoreType, MachinePointerInfo(), PartAlign,
            MachineMemOperand::MOStore);
        InGlue = Chain.getValue(1);
 
        // Cleanup.
        StoreOperands.clear();
 
        // TODO: We may need to support vector types that can be passed
        // as scalars in variadic arguments.
        if (!IsByVal && IsVAArg) {
          assert(NumElts == 1 &&
                 "Vectorization is expected to be disabled for variadics.");
          VAOffset += DL.getTypeAllocSize(
              TheStoreType.getTypeForEVT(*DAG.getContext()));
        }
      }
      if (!IsByVal)
        ++OIdx;
    }
    assert(StoreOperands.empty() && "Unfinished parameter store.");
    if (!IsByVal && VTs.size() > 0)
      --OIdx;
    ++ParamCount;
    if (IsByVal && IsVAArg)
      VAOffset += TypeSize;
  }
 
  GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
  MaybeAlign retAlignment = std::nullopt;
 
  // Handle Result
  if (Ins.size() > 0) {
    SmallVector<EVT, 16> resvtparts;
    ComputeValueVTs(*this, DL, RetTy, resvtparts);
 
    // Declare
    //  .param .align N .b8 retval0[<size-in-bytes>], or
    //  .param .b<size-in-bits> retval0
    unsigned resultsz = DL.getTypeAllocSizeInBits(RetTy);
    if (!IsTypePassedAsArray(RetTy)) {
      resultsz = promoteScalarArgumentSize(resultsz);
      SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
      SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
                                  DAG.getConstant(resultsz, dl, MVT::i32),
                                  DAG.getConstant(0, dl, MVT::i32), InGlue };
      Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,
                          DeclareRetOps);
      InGlue = Chain.getValue(1);
    } else {
      retAlignment = getArgumentAlignment(Callee, CB, RetTy, 0, DL);
      assert(retAlignment && "retAlignment is guaranteed to be set");
      SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
      SDValue DeclareRetOps[] = {
          Chain, DAG.getConstant(retAlignment->value(), dl, MVT::i32),
          DAG.getConstant(resultsz / 8, dl, MVT::i32),
          DAG.getConstant(0, dl, MVT::i32), InGlue};
      Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
                          DeclareRetOps);
      InGlue = Chain.getValue(1);
    }
  }
 
  bool HasVAArgs = CLI.IsVarArg && (CLI.Args.size() > CLI.NumFixedArgs);
  // Set the size of the vararg param byte array if the callee is a variadic
  // function and the variadic part is not empty.
  if (HasVAArgs) {
    SDValue DeclareParamOps[] = {
        VADeclareParam.getOperand(0), VADeclareParam.getOperand(1),
        VADeclareParam.getOperand(2), DAG.getConstant(VAOffset, dl, MVT::i32),
        VADeclareParam.getOperand(4)};
    DAG.MorphNodeTo(VADeclareParam.getNode(), VADeclareParam.getOpcode(),
                    VADeclareParam->getVTList(), DeclareParamOps);
  }
 
  // Both indirect calls and libcalls have nullptr Func. In order to distinguish
  // between them we must rely on the call site value which is valid for
  // indirect calls but is always null for libcalls.
  bool isIndirectCall = !Func && CB;
 
  if (isa<ExternalSymbolSDNode>(Callee)) {
    Function* CalleeFunc = nullptr;
 
    // Try to find the callee in the current module.
    Callee = DAG.getSymbolFunctionGlobalAddress(Callee, &CalleeFunc);
    assert(CalleeFunc != nullptr && "Libcall callee must be set.");
 
    // Set the "libcall callee" attribute to indicate that the function
    // must always have a declaration.
    CalleeFunc->addFnAttr("nvptx-libcall-callee", "true");
  }
 
  if (isIndirectCall) {
    // This is indirect function call case : PTX requires a prototype of the
    // form
    // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
    // to be emitted, and the label has to used as the last arg of call
    // instruction.
    // The prototype is embedded in a string and put as the operand for a
    // CallPrototype SDNode which will print out to the value of the string.
    SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue);
    std::string Proto = getPrototype(
        DL, RetTy, Args, Outs, retAlignment,
        HasVAArgs
            ? std::optional<std::pair<unsigned, const APInt &>>(std::make_pair(
                  CLI.NumFixedArgs,
                  cast<ConstantSDNode>(VADeclareParam->getOperand(1))
                      ->getAPIntValue()))
            : std::nullopt,
        *CB, UniqueCallSite);
    const char *ProtoStr = nvTM->getStrPool().save(Proto).data();
    SDValue ProtoOps[] = {
        Chain,
        DAG.getTargetExternalSymbol(ProtoStr, MVT::i32),
        InGlue,
    };
    Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, ProtoVTs, ProtoOps);
    InGlue = Chain.getValue(1);
  }
  // Op to just print "call"
  SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue PrintCallOps[] = {
    Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, dl, MVT::i32), InGlue
  };
  // We model convergent calls as separate opcodes.
  unsigned Opcode = isIndirectCall ? NVPTXISD::PrintCall : NVPTXISD::PrintCallUni;
  if (CLI.IsConvergent)
    Opcode = Opcode == NVPTXISD::PrintCallUni ? NVPTXISD::PrintConvergentCallUni
                                              : NVPTXISD::PrintConvergentCall;
  Chain = DAG.getNode(Opcode, dl, PrintCallVTs, PrintCallOps);
  InGlue = Chain.getValue(1);
 
  // Ops to print out the function name
  SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue CallVoidOps[] = { Chain, Callee, InGlue };
  Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps);
  InGlue = Chain.getValue(1);
 
  // Ops to print out the param list
  SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue CallArgBeginOps[] = { Chain, InGlue };
  Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs,
                      CallArgBeginOps);
  InGlue = Chain.getValue(1);
 
  for (unsigned i = 0, e = std::min(CLI.NumFixedArgs + 1, ParamCount); i != e;
       ++i) {
    unsigned opcode;
    if (i == (e - 1))
      opcode = NVPTXISD::LastCallArg;
    else
      opcode = NVPTXISD::CallArg;
    SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);
    SDValue CallArgOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
                             DAG.getConstant(i, dl, MVT::i32), InGlue };
    Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps);
    InGlue = Chain.getValue(1);
  }
  SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue CallArgEndOps[] = { Chain,
                              DAG.getConstant(isIndirectCall ? 0 : 1, dl, MVT::i32),
                              InGlue };
  Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps);
  InGlue = Chain.getValue(1);
 
  if (isIndirectCall) {
    SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);
    SDValue PrototypeOps[] = {
        Chain, DAG.getConstant(UniqueCallSite, dl, MVT::i32), InGlue};
    Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps);
    InGlue = Chain.getValue(1);
  }
 
  SmallVector<SDValue, 16> ProxyRegOps;
  SmallVector<std::optional<MVT>, 16> ProxyRegTruncates;
 
  // Generate loads from param memory/moves from registers for result
  if (Ins.size() > 0) {
    SmallVector<EVT, 16> VTs;
    SmallVector<uint64_t, 16> Offsets;
    ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets, 0);
    assert(VTs.size() == Ins.size() && "Bad value decomposition");
 
    Align RetAlign = getArgumentAlignment(Callee, CB, RetTy, 0, DL);
    auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign);
 
    SmallVector<EVT, 6> LoadVTs;
    int VecIdx = -1; // Index of the first element of the vector.
 
    // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
    // 32-bits are sign extended or zero extended, depending on whether
    // they are signed or unsigned types.
    bool ExtendIntegerRetVal =
        RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
 
    for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
      bool needTruncate = false;
      EVT TheLoadType = VTs[i];
      EVT EltType = Ins[i].VT;
      Align EltAlign = commonAlignment(RetAlign, Offsets[i]);
      MVT PromotedVT;
 
      if (PromoteScalarIntegerPTX(TheLoadType, &PromotedVT)) {
        TheLoadType = EVT(PromotedVT);
        EltType = EVT(PromotedVT);
        needTruncate = true;
      }
 
      if (ExtendIntegerRetVal) {
        TheLoadType = MVT::i32;
        EltType = MVT::i32;
        needTruncate = true;
      } else if (TheLoadType.getSizeInBits() < 16) {
        if (VTs[i].isInteger())
          needTruncate = true;
        EltType = MVT::i16;
      }
 
      // Record index of the very first element of the vector.
      if (VectorInfo[i] & PVF_FIRST) {
        assert(VecIdx == -1 && LoadVTs.empty() && "Orphaned operand list.");
        VecIdx = i;
      }
 
      LoadVTs.push_back(EltType);
 
      if (VectorInfo[i] & PVF_LAST) {
        unsigned NumElts = LoadVTs.size();
        LoadVTs.push_back(MVT::Other);
        LoadVTs.push_back(MVT::Glue);
        NVPTXISD::NodeType Op;
        switch (NumElts) {
        case 1:
          Op = NVPTXISD::LoadParam;
          break;
        case 2:
          Op = NVPTXISD::LoadParamV2;
          break;
        case 4:
          Op = NVPTXISD::LoadParamV4;
          break;
        default:
          llvm_unreachable("Invalid vector info.");
        }
 
        SDValue LoadOperands[] = {
            Chain, DAG.getConstant(1, dl, MVT::i32),
            DAG.getConstant(Offsets[VecIdx], dl, MVT::i32), InGlue};
        SDValue RetVal = DAG.getMemIntrinsicNode(
            Op, dl, DAG.getVTList(LoadVTs), LoadOperands, TheLoadType,
            MachinePointerInfo(), EltAlign,
            MachineMemOperand::MOLoad);
 
        for (unsigned j = 0; j < NumElts; ++j) {
          ProxyRegOps.push_back(RetVal.getValue(j));
 
          if (needTruncate)
            ProxyRegTruncates.push_back(std::optional<MVT>(Ins[VecIdx + j].VT));
          else
            ProxyRegTruncates.push_back(std::optional<MVT>());
        }
 
        Chain = RetVal.getValue(NumElts);
        InGlue = RetVal.getValue(NumElts + 1);
 
        // Cleanup
        VecIdx = -1;
        LoadVTs.clear();
      }
    }
  }
 
  Chain =
      DAG.getCALLSEQ_END(Chain, UniqueCallSite, UniqueCallSite + 1, InGlue, dl);
  InGlue = Chain.getValue(1);
 
  // Append ProxyReg instructions to the chain to make sure that `callseq_end`
  // will not get lost. Otherwise, during libcalls expansion, the nodes can become
  // dangling.
  for (unsigned i = 0; i < ProxyRegOps.size(); ++i) {
    SDValue Ret = DAG.getNode(
      NVPTXISD::ProxyReg, dl,
      DAG.getVTList(ProxyRegOps[i].getSimpleValueType(), MVT::Other, MVT::Glue),
      { Chain, ProxyRegOps[i], InGlue }
    );
 
    Chain = Ret.getValue(1);
    InGlue = Ret.getValue(2);
 
    if (ProxyRegTruncates[i]) {
      Ret = DAG.getNode(ISD::TRUNCATE, dl, *ProxyRegTruncates[i], Ret);
    }
 
    InVals.push_back(Ret);
  }
 
  // set isTailCall to false for now, until we figure out how to express
  // tail call optimization in PTX
  isTailCall = false;
  return Chain;
}
 
// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
// (see LegalizeDAG.cpp). This is slow and uses local memory.
// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
SDValue
NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
  SDNode *Node = Op.getNode();
  SDLoc dl(Node);
  SmallVector<SDValue, 8> Ops;
  unsigned NumOperands = Node->getNumOperands();
  for (unsigned i = 0; i < NumOperands; ++i) {
    SDValue SubOp = Node->getOperand(i);
    EVT VVT = SubOp.getNode()->getValueType(0);
    EVT EltVT = VVT.getVectorElementType();
    unsigned NumSubElem = VVT.getVectorNumElements();
    for (unsigned j = 0; j < NumSubElem; ++j) {
      Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
                                DAG.getIntPtrConstant(j, dl)));
    }
  }
  return DAG.getBuildVector(Node->getValueType(0), dl, Ops);
}
 
// We can init constant f16x2 with a single .b32 move.  Normally it
// would get lowered as two constant loads and vector-packing move.
//        mov.b16         %h1, 0x4000;
//        mov.b16         %h2, 0x3C00;
//        mov.b32         %hh2, {%h2, %h1};
// Instead we want just a constant move:
//        mov.b32         %hh2, 0x40003C00
//
// This results in better SASS code with CUDA 7.x. Ptxas in CUDA 8.0
// generates good SASS in both cases.
SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                               SelectionDAG &DAG) const {
  EVT VT = Op->getValueType(0);
  if (!(Isv2x16VT(VT)))
    return Op;
  APInt E0;
  APInt E1;
  if (VT == MVT::v2f16 || VT == MVT::v2bf16) {
    if (!(isa<ConstantFPSDNode>(Op->getOperand(0)) &&
          isa<ConstantFPSDNode>(Op->getOperand(1))))
      return Op;
 
    E0 = cast<ConstantFPSDNode>(Op->getOperand(0))
             ->getValueAPF()
             .bitcastToAPInt();
    E1 = cast<ConstantFPSDNode>(Op->getOperand(1))
             ->getValueAPF()
             .bitcastToAPInt();
  } else {
    assert(VT == MVT::v2i16);
    if (!(isa<ConstantSDNode>(Op->getOperand(0)) &&
          isa<ConstantSDNode>(Op->getOperand(1))))
      return Op;
 
    E0 = cast<ConstantSDNode>(Op->getOperand(0))->getAPIntValue();
    E1 = cast<ConstantSDNode>(Op->getOperand(1))->getAPIntValue();
  }
  SDValue Const =
      DAG.getConstant(E1.zext(32).shl(16) | E0.zext(32), SDLoc(Op), MVT::i32);
  return DAG.getNode(ISD::BITCAST, SDLoc(Op), Op->getValueType(0), Const);
}
 
SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                     SelectionDAG &DAG) const {
  SDValue Index = Op->getOperand(1);
  // Constant index will be matched by tablegen.
  if (isa<ConstantSDNode>(Index.getNode()))
    return Op;
 
  // Extract individual elements and select one of them.
  SDValue Vector = Op->getOperand(0);
  EVT VectorVT = Vector.getValueType();
  assert(Isv2x16VT(VectorVT) && "Unexpected vector type.");
  EVT EltVT = VectorVT.getVectorElementType();
 
  SDLoc dl(Op.getNode());
  SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,
                           DAG.getIntPtrConstant(0, dl));
  SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,
                           DAG.getIntPtrConstant(1, dl));
  return DAG.getSelectCC(dl, Index, DAG.getIntPtrConstant(0, dl), E0, E1,
                         ISD::CondCode::SETEQ);
}
 
/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which
/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
///    amount, or
/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
///    amount.
SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
                                                  SelectionDAG &DAG) const {
  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
  assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
 
  EVT VT = Op.getValueType();
  unsigned VTBits = VT.getSizeInBits();
  SDLoc dl(Op);
  SDValue ShOpLo = Op.getOperand(0);
  SDValue ShOpHi = Op.getOperand(1);
  SDValue ShAmt  = Op.getOperand(2);
  unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
 
  if (VTBits == 32 && STI.getSmVersion() >= 35) {
    // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
    // {dHi, dLo} = {aHi, aLo} >> Amt
    //   dHi = aHi >> Amt
    //   dLo = shf.r.clamp aLo, aHi, Amt
 
    SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
    SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi,
                             ShAmt);
 
    SDValue Ops[2] = { Lo, Hi };
    return DAG.getMergeValues(Ops, dl);
  }
  else {
    // {dHi, dLo} = {aHi, aLo} >> Amt
    // - if (Amt>=size) then
    //      dLo = aHi >> (Amt-size)
    //      dHi = aHi >> Amt (this is either all 0 or all 1)
    //   else
    //      dLo = (aLo >>logic Amt) | (aHi << (size-Amt))
    //      dHi = aHi >> Amt
 
    SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
                                   DAG.getConstant(VTBits, dl, MVT::i32),
                                   ShAmt);
    SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
    SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
                                     DAG.getConstant(VTBits, dl, MVT::i32));
    SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
    SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
    SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
 
    SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
                               DAG.getConstant(VTBits, dl, MVT::i32),
                               ISD::SETGE);
    SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
    SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
 
    SDValue Ops[2] = { Lo, Hi };
    return DAG.getMergeValues(Ops, dl);
  }
}
 
/// LowerShiftLeftParts - Lower SHL_PARTS, which
/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
///    amount, or
/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
///    amount.
SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
                                                 SelectionDAG &DAG) const {
  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
  assert(Op.getOpcode() == ISD::SHL_PARTS);
 
  EVT VT = Op.getValueType();
  unsigned VTBits = VT.getSizeInBits();
  SDLoc dl(Op);
  SDValue ShOpLo = Op.getOperand(0);
  SDValue ShOpHi = Op.getOperand(1);
  SDValue ShAmt  = Op.getOperand(2);
 
  if (VTBits == 32 && STI.getSmVersion() >= 35) {
    // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
    // {dHi, dLo} = {aHi, aLo} << Amt
    //   dHi = shf.l.clamp aLo, aHi, Amt
    //   dLo = aLo << Amt
 
    SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi,
                             ShAmt);
    SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
 
    SDValue Ops[2] = { Lo, Hi };
    return DAG.getMergeValues(Ops, dl);
  }
  else {
    // {dHi, dLo} = {aHi, aLo} << Amt
    // - if (Amt>=size) then
    //      dLo = aLo << Amt (all 0)
    //      dLo = aLo << (Amt-size)
    //   else
    //      dLo = aLo << Amt
    //      dHi = (aHi << Amt) | (aLo >> (size-Amt))
 
    SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
                                   DAG.getConstant(VTBits, dl, MVT::i32),
                                   ShAmt);
    SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
    SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
                                     DAG.getConstant(VTBits, dl, MVT::i32));
    SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
    SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
    SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
 
    SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
                               DAG.getConstant(VTBits, dl, MVT::i32),
                               ISD::SETGE);
    SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
    SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
 
    SDValue Ops[2] = { Lo, Hi };
    return DAG.getMergeValues(Ops, dl);
  }
}
 
SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
 
  if (VT == MVT::f32)
    return LowerFROUND32(Op, DAG);
 
  if (VT == MVT::f64)
    return LowerFROUND64(Op, DAG);
 
  llvm_unreachable("unhandled type");
}
 
// This is the the rounding method used in CUDA libdevice in C like code:
// float roundf(float A)
// {
//   float RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f));
//   RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
//   return abs(A) < 0.5 ? (float)(int)A : RoundedA;
// }
SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
                                           SelectionDAG &DAG) const {
  SDLoc SL(Op);
  SDValue A = Op.getOperand(0);
  EVT VT = Op.getValueType();
 
  SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);
 
  // RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f))
  SDValue Bitcast  = DAG.getNode(ISD::BITCAST, SL, MVT::i32, A);
  const int SignBitMask = 0x80000000;
  SDValue Sign = DAG.getNode(ISD::AND, SL, MVT::i32, Bitcast,
                             DAG.getConstant(SignBitMask, SL, MVT::i32));
  const int PointFiveInBits = 0x3F000000;
  SDValue PointFiveWithSignRaw =
      DAG.getNode(ISD::OR, SL, MVT::i32, Sign,
                  DAG.getConstant(PointFiveInBits, SL, MVT::i32));
  SDValue PointFiveWithSign =
      DAG.getNode(ISD::BITCAST, SL, VT, PointFiveWithSignRaw);
  SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, A, PointFiveWithSign);
  SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);
 
  // RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue IsLarge =
      DAG.getSetCC(SL, SetCCVT, AbsA, DAG.getConstantFP(pow(2.0, 23.0), SL, VT),
                   ISD::SETOGT);
  RoundedA = DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA);
 
  // return abs(A) < 0.5 ? (float)(int)A : RoundedA;
  SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,
                                DAG.getConstantFP(0.5, SL, VT), ISD::SETOLT);
  SDValue RoundedAForSmallA = DAG.getNode(ISD::FTRUNC, SL, VT, A);
  return DAG.getNode(ISD::SELECT, SL, VT, IsSmall, RoundedAForSmallA, RoundedA);
}
 
// The implementation of round(double) is similar to that of round(float) in
// that they both separate the value range into three regions and use a method
// specific to the region to round the values. However, round(double) first
// calculates the round of the absolute value and then adds the sign back while
// round(float) directly rounds the value with sign.
SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
                                           SelectionDAG &DAG) const {
  SDLoc SL(Op);
  SDValue A = Op.getOperand(0);
  EVT VT = Op.getValueType();
 
  SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);
 
  // double RoundedA = (double) (int) (abs(A) + 0.5f);
  SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, AbsA,
                                  DAG.getConstantFP(0.5, SL, VT));
  SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);
 
  // RoundedA = abs(A) < 0.5 ? (double)0 : RoundedA;
  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,
                                DAG.getConstantFP(0.5, SL, VT), ISD::SETOLT);
  RoundedA = DAG.getNode(ISD::SELECT, SL, VT, IsSmall,
                         DAG.getConstantFP(0, SL, VT),
                         RoundedA);
 
  // Add sign to rounded_A
  RoundedA = DAG.getNode(ISD::FCOPYSIGN, SL, VT, RoundedA, A);
  DAG.getNode(ISD::FTRUNC, SL, VT, A);
 
  // RoundedA = abs(A) > 0x1.0p52 ? A : RoundedA;
  SDValue IsLarge =
      DAG.getSetCC(SL, SetCCVT, AbsA, DAG.getConstantFP(pow(2.0, 52.0), SL, VT),
                   ISD::SETOGT);
  return DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA);
}
 
static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  if (Op.getValueType() != MVT::v2i16)
    return Op;
  EVT EltVT = Op.getValueType().getVectorElementType();
  SmallVector<SDValue> VecElements;
  for (int I = 0, E = Op.getValueType().getVectorNumElements(); I < E; I++) {
    SmallVector<SDValue> ScalarArgs;
    llvm::transform(Op->ops(), std::back_inserter(ScalarArgs),
                    [&](const SDUse &O) {
                      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
                                         O.get(), DAG.getIntPtrConstant(I, DL));
                    });
    VecElements.push_back(DAG.getNode(Op.getOpcode(), DL, EltVT, ScalarArgs));
  }
  SDValue V =
      DAG.getNode(ISD::BUILD_VECTOR, DL, Op.getValueType(), VecElements);
  return V;
}
 
SDValue
NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
  switch (Op.getOpcode()) {
  case ISD::RETURNADDR:
    return SDValue();
  case ISD::FRAMEADDR:
    return SDValue();
  case ISD::GlobalAddress:
    return LowerGlobalAddress(Op, DAG);
  case ISD::INTRINSIC_W_CHAIN:
    return Op;
  case ISD::BUILD_VECTOR:
    return LowerBUILD_VECTOR(Op, DAG);
  case ISD::EXTRACT_SUBVECTOR:
    return Op;
  case ISD::EXTRACT_VECTOR_ELT:
    return LowerEXTRACT_VECTOR_ELT(Op, DAG);
  case ISD::CONCAT_VECTORS:
    return LowerCONCAT_VECTORS(Op, DAG);
  case ISD::STORE:
    return LowerSTORE(Op, DAG);
  case ISD::LOAD:
    return LowerLOAD(Op, DAG);
  case ISD::SHL_PARTS:
    return LowerShiftLeftParts(Op, DAG);
  case ISD::SRA_PARTS:
  case ISD::SRL_PARTS:
    return LowerShiftRightParts(Op, DAG);
  case ISD::SELECT:
    return LowerSelect(Op, DAG);
  case ISD::FROUND:
    return LowerFROUND(Op, DAG);
  case ISD::VAARG:
    return LowerVAARG(Op, DAG);
  case ISD::VASTART:
    return LowerVASTART(Op, DAG);
  case ISD::ABS:
  case ISD::SMIN:
  case ISD::SMAX:
  case ISD::UMIN:
  case ISD::UMAX:
  case ISD::ADD:
  case ISD::SUB:
  case ISD::MUL:
  case ISD::SHL:
  case ISD::SREM:
  case ISD::UREM:
    return LowerVectorArith(Op, DAG);
  default:
    llvm_unreachable("Custom lowering not defined for operation");
  }
}
 
// This function is almost a copy of SelectionDAG::expandVAArg().
// The only diff is that this one produces loads from local address space.
SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
  const TargetLowering *TLI = STI.getTargetLowering();
  SDLoc DL(Op);
 
  SDNode *Node = Op.getNode();
  const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
  EVT VT = Node->getValueType(0);
  auto *Ty = VT.getTypeForEVT(*DAG.getContext());
  SDValue Tmp1 = Node->getOperand(0);
  SDValue Tmp2 = Node->getOperand(1);
  const MaybeAlign MA(Node->getConstantOperandVal(3));
 
  SDValue VAListLoad = DAG.getLoad(TLI->getPointerTy(DAG.getDataLayout()), DL,
                                   Tmp1, Tmp2, MachinePointerInfo(V));
  SDValue VAList = VAListLoad;
 
  if (MA && *MA > TLI->getMinStackArgumentAlignment()) {
    VAList = DAG.getNode(
        ISD::ADD, DL, VAList.getValueType(), VAList,
        DAG.getConstant(MA->value() - 1, DL, VAList.getValueType()));
 
    VAList = DAG.getNode(
        ISD::AND, DL, VAList.getValueType(), VAList,
        DAG.getConstant(-(int64_t)MA->value(), DL, VAList.getValueType()));
  }
 
  // Increment the pointer, VAList, to the next vaarg
  Tmp1 = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
                     DAG.getConstant(DAG.getDataLayout().getTypeAllocSize(Ty),
                                     DL, VAList.getValueType()));
 
  // Store the incremented VAList to the legalized pointer
  Tmp1 = DAG.getStore(VAListLoad.getValue(1), DL, Tmp1, Tmp2,
                      MachinePointerInfo(V));
 
  const Value *SrcV =
      Constant::getNullValue(PointerType::get(Ty, ADDRESS_SPACE_LOCAL));
 
  // Load the actual argument out of the pointer VAList
  return DAG.getLoad(VT, DL, Tmp1, VAList, MachinePointerInfo(SrcV));
}
 
SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
  const TargetLowering *TLI = STI.getTargetLowering();
  SDLoc DL(Op);
  EVT PtrVT = TLI->getPointerTy(DAG.getDataLayout());
 
  // Store the address of unsized array <function>_vararg[] in the ap object.
  SDValue Arg = getParamSymbol(DAG, /* vararg */ -1, PtrVT);
  SDValue VAReg = DAG.getNode(NVPTXISD::Wrapper, DL, PtrVT, Arg);
 
  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
  return DAG.getStore(Op.getOperand(0), DL, VAReg, Op.getOperand(1),
                      MachinePointerInfo(SV));
}
 
SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const {
  SDValue Op0 = Op->getOperand(0);
  SDValue Op1 = Op->getOperand(1);
  SDValue Op2 = Op->getOperand(2);
  SDLoc DL(Op.getNode());
 
  assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
 
  Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
  Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
  SDValue Select = DAG.getNode(ISD::SELECT, DL, MVT::i32, Op0, Op1, Op2);
  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);
 
  return Trunc;
}
 
SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
  if (Op.getValueType() == MVT::i1)
    return LowerLOADi1(Op, DAG);
 
  // v2f16/v2bf16/v2i16 are legal, so we can't rely on legalizer to handle
  // unaligned loads and have to handle it here.
  if (Isv2x16VT(Op.getValueType())) {
    LoadSDNode *Load = cast<LoadSDNode>(Op);
    EVT MemVT = Load->getMemoryVT();
    if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                        MemVT, *Load->getMemOperand())) {
      SDValue Ops[2];
      std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG);
      return DAG.getMergeValues(Ops, SDLoc(Op));
    }
  }
 
  return SDValue();
}
 
// v = ld i1* addr
//   =>
// v1 = ld i8* addr (-> i16)
// v = trunc i16 to i1
SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
  SDNode *Node = Op.getNode();
  LoadSDNode *LD = cast<LoadSDNode>(Node);
  SDLoc dl(Node);
  assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
  assert(Node->getValueType(0) == MVT::i1 &&
         "Custom lowering for i1 load only");
  SDValue newLD = DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),
                              LD->getPointerInfo(), LD->getAlign(),
                              LD->getMemOperand()->getFlags());
  SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
  // The legalizer (the caller) is expecting two values from the legalized
  // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
  // in LegalizeDAG.cpp which also uses MergeValues.
  SDValue Ops[] = { result, LD->getChain() };
  return DAG.getMergeValues(Ops, dl);
}
 
SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
  StoreSDNode *Store = cast<StoreSDNode>(Op);
  EVT VT = Store->getMemoryVT();
 
  if (VT == MVT::i1)
    return LowerSTOREi1(Op, DAG);
 
  // v2f16 is legal, so we can't rely on legalizer to handle unaligned
  // stores and have to handle it here.
  if (Isv2x16VT(VT) &&
      !allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                      VT, *Store->getMemOperand()))
    return expandUnalignedStore(Store, DAG);
 
  // v2f16, v2bf16 and v2i16 don't need special handling.
  if (Isv2x16VT(VT))
    return SDValue();
 
  if (VT.isVector())
    return LowerSTOREVector(Op, DAG);
 
  return SDValue();
}
 
SDValue
NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
  SDNode *N = Op.getNode();
  SDValue Val = N->getOperand(1);
  SDLoc DL(N);
  EVT ValVT = Val.getValueType();
 
  if (ValVT.isVector()) {
    // We only handle "native" vector sizes for now, e.g. <4 x double> is not
    // legal.  We can (and should) split that into 2 stores of <2 x double> here
    // but I'm leaving that as a TODO for now.
    if (!ValVT.isSimple())
      return SDValue();
    switch (ValVT.getSimpleVT().SimpleTy) {
    default:
      return SDValue();
    case MVT::v2i8:
    case MVT::v2i16:
    case MVT::v2i32:
    case MVT::v2i64:
    case MVT::v2f16:
    case MVT::v2bf16:
    case MVT::v2f32:
    case MVT::v2f64:
    case MVT::v4i8:
    case MVT::v4i16:
    case MVT::v4i32:
    case MVT::v4f16:
    case MVT::v4bf16:
    case MVT::v4f32:
    case MVT::v8f16: // <4 x f16x2>
    case MVT::v8bf16: // <4 x bf16x2>
    case MVT::v8i16:  // <4 x i16x2>
      // This is a "native" vector type
      break;
    }
 
    MemSDNode *MemSD = cast<MemSDNode>(N);
    const DataLayout &TD = DAG.getDataLayout();
 
    Align Alignment = MemSD->getAlign();
    Align PrefAlign =
        TD.getPrefTypeAlign(ValVT.getTypeForEVT(*DAG.getContext()));
    if (Alignment < PrefAlign) {
      // This store is not sufficiently aligned, so bail out and let this vector
      // store be scalarized.  Note that we may still be able to emit smaller
      // vector stores.  For example, if we are storing a <4 x float> with an
      // alignment of 8, this check will fail but the legalizer will try again
      // with 2 x <2 x float>, which will succeed with an alignment of 8.
      return SDValue();
    }
 
    unsigned Opcode = 0;
    EVT EltVT = ValVT.getVectorElementType();
    unsigned NumElts = ValVT.getVectorNumElements();
 
    // Since StoreV2 is a target node, we cannot rely on DAG type legalization.
    // Therefore, we must ensure the type is legal.  For i1 and i8, we set the
    // stored type to i16 and propagate the "real" type as the memory type.
    bool NeedExt = false;
    if (EltVT.getSizeInBits() < 16)
      NeedExt = true;
 
    bool StoreF16x2 = false;
    switch (NumElts) {
    default:
      return SDValue();
    case 2:
      Opcode = NVPTXISD::StoreV2;
      break;
    case 4:
      Opcode = NVPTXISD::StoreV4;
      break;
    case 8:
      // v8f16 is a special case. PTX doesn't have st.v8.f16
      // instruction. Instead, we split the vector into v2f16 chunks and
      // store them with st.v4.b32.
      assert(Is16bitsType(EltVT.getSimpleVT()) && "Wrong type for the vector.");
      Opcode = NVPTXISD::StoreV4;
      StoreF16x2 = true;
      break;
    }
 
    SmallVector<SDValue, 8> Ops;
 
    // First is the chain
    Ops.push_back(N->getOperand(0));
 
    if (StoreF16x2) {
      // Combine f16,f16 -> v2f16
      NumElts /= 2;
      for (unsigned i = 0; i < NumElts; ++i) {
        SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
                                 DAG.getIntPtrConstant(i * 2, DL));
        SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
                                 DAG.getIntPtrConstant(i * 2 + 1, DL));
        EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 2);
        SDValue V2 = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, E0, E1);
        Ops.push_back(V2);
      }
    } else {
      // Then the split values
      for (unsigned i = 0; i < NumElts; ++i) {
        SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
                                     DAG.getIntPtrConstant(i, DL));
        if (NeedExt)
          ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);
        Ops.push_back(ExtVal);
      }
    }
 
    // Then any remaining arguments
    Ops.append(N->op_begin() + 2, N->op_end());
 
    SDValue NewSt =
        DAG.getMemIntrinsicNode(Opcode, DL, DAG.getVTList(MVT::Other), Ops,
                                MemSD->getMemoryVT(), MemSD->getMemOperand());
 
    // return DCI.CombineTo(N, NewSt, true);
    return NewSt;
  }
 
  return SDValue();
}
 
// st i1 v, addr
//    =>
// v1 = zxt v to i16
// st.u8 i16, addr
SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
  SDNode *Node = Op.getNode();
  SDLoc dl(Node);
  StoreSDNode *ST = cast<StoreSDNode>(Node);
  SDValue Tmp1 = ST->getChain();
  SDValue Tmp2 = ST->getBasePtr();
  SDValue Tmp3 = ST->getValue();
  assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
  Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);
  SDValue Result =
      DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), MVT::i8,
                        ST->getAlign(), ST->getMemOperand()->getFlags());
  return Result;
}
 
// This creates target external symbol for a function parameter.
// Name of the symbol is composed from its index and the function name.
// Negative index corresponds to special parameter (unsized array) used for
// passing variable arguments.
SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx,
                                            EVT v) const {
  StringRef SavedStr = nvTM->getStrPool().save(
      getParamName(&DAG.getMachineFunction().getFunction(), idx));
  return DAG.getTargetExternalSymbol(SavedStr.data(), v);
}
 
SDValue NVPTXTargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  MachineFunction &MF = DAG.getMachineFunction();
  const DataLayout &DL = DAG.getDataLayout();
  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
  const Function *F = &MF.getFunction();
  const AttributeList &PAL = F->getAttributes();
  const TargetLowering *TLI = STI.getTargetLowering();
 
  SDValue Root = DAG.getRoot();
  std::vector<SDValue> OutChains;
 
  bool isABI = (STI.getSmVersion() >= 20);
  assert(isABI && "Non-ABI compilation is not supported");
  if (!isABI)
    return Chain;
 
  std::vector<Type *> argTypes;
  std::vector<const Argument *> theArgs;
  for (const Argument &I : F->args()) {
    theArgs.push_back(&I);
    argTypes.push_back(I.getType());
  }
  // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
  // Ins.size() will be larger
  //   * if there is an aggregate argument with multiple fields (each field
  //     showing up separately in Ins)
  //   * if there is a vector argument with more than typical vector-length
  //     elements (generally if more than 4) where each vector element is
  //     individually present in Ins.
  // So a different index should be used for indexing into Ins.
  // See similar issue in LowerCall.
  unsigned InsIdx = 0;
 
  int idx = 0;
  for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++idx, ++InsIdx) {
    Type *Ty = argTypes[i];
 
    if (theArgs[i]->use_empty()) {
      // argument is dead
      if (IsTypePassedAsArray(Ty) && !Ty->isVectorTy()) {
        SmallVector<EVT, 16> vtparts;
 
        ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts);
        if (vtparts.empty())
          report_fatal_error("Empty parameter types are not supported");
 
        for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
             ++parti) {
          InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
          ++InsIdx;
        }
        if (vtparts.size() > 0)
          --InsIdx;
        continue;
      }
      if (Ty->isVectorTy()) {
        EVT ObjectVT = getValueType(DL, Ty);
        unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);
        for (unsigned parti = 0; parti < NumRegs; ++parti) {
          InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
          ++InsIdx;
        }
        if (NumRegs > 0)
          --InsIdx;
        continue;
      }
      InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
      continue;
    }
 
    // In the following cases, assign a node order of "idx+1"
    // to newly created nodes. The SDNodes for params have to
    // appear in the same order as their order of appearance
    // in the original function. "idx+1" holds that order.
    if (!PAL.hasParamAttr(i, Attribute::ByVal)) {
      bool aggregateIsPacked = false;
      if (StructType *STy = dyn_cast<StructType>(Ty))
        aggregateIsPacked = STy->isPacked();
 
      SmallVector<EVT, 16> VTs;
      SmallVector<uint64_t, 16> Offsets;
      ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets, 0);
      if (VTs.empty())
        report_fatal_error("Empty parameter types are not supported");
 
      auto VectorInfo =
          VectorizePTXValueVTs(VTs, Offsets, DL.getABITypeAlign(Ty));
 
      SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
      int VecIdx = -1; // Index of the first element of the current vector.
      for (unsigned parti = 0, parte = VTs.size(); parti != parte; ++parti) {
        if (VectorInfo[parti] & PVF_FIRST) {
          assert(VecIdx == -1 && "Orphaned vector.");
          VecIdx = parti;
        }
 
        // That's the last element of this store op.
        if (VectorInfo[parti] & PVF_LAST) {
          unsigned NumElts = parti - VecIdx + 1;
          EVT EltVT = VTs[parti];
          // i1 is loaded/stored as i8.
          EVT LoadVT = EltVT;
          if (EltVT == MVT::i1)
            LoadVT = MVT::i8;
          else if (Isv2x16VT(EltVT))
            // getLoad needs a vector type, but it can't handle
            // vectors which contain v2f16 or v2bf16 elements. So we must load
            // using i32 here and then bitcast back.
            LoadVT = MVT::i32;
 
          EVT VecVT = EVT::getVectorVT(F->getContext(), LoadVT, NumElts);
          SDValue VecAddr =
              DAG.getNode(ISD::ADD, dl, PtrVT, Arg,
                          DAG.getConstant(Offsets[VecIdx], dl, PtrVT));
          Value *srcValue = Constant::getNullValue(PointerType::get(
              EltVT.getTypeForEVT(F->getContext()), ADDRESS_SPACE_PARAM));
          SDValue P = DAG.getLoad(VecVT, dl, Root, VecAddr,
                                  MachinePointerInfo(srcValue),
                                  MaybeAlign(aggregateIsPacked ? 1 : 0),
                                  MachineMemOperand::MODereferenceable |
                                      MachineMemOperand::MOInvariant);
          if (P.getNode())
            P.getNode()->setIROrder(idx + 1);
          for (unsigned j = 0; j < NumElts; ++j) {
            SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LoadVT, P,
                                      DAG.getIntPtrConstant(j, dl));
            // We've loaded i1 as an i8 and now must truncate it back to i1
            if (EltVT == MVT::i1)
              Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Elt);
            // v2f16 was loaded as an i32. Now we must bitcast it back.
            else if (Isv2x16VT(EltVT))
              Elt = DAG.getNode(ISD::BITCAST, dl, EltVT, Elt);
 
            // If a promoted integer type is used, truncate down to the original
            MVT PromotedVT;
            if (PromoteScalarIntegerPTX(EltVT, &PromotedVT)) {
              Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
            }
 
            // Extend the element if necessary (e.g. an i8 is loaded
            // into an i16 register)
            if (Ins[InsIdx].VT.isInteger() &&
                Ins[InsIdx].VT.getFixedSizeInBits() >
                    LoadVT.getFixedSizeInBits()) {
              unsigned Extend = Ins[InsIdx].Flags.isSExt() ? ISD::SIGN_EXTEND
                                                           : ISD::ZERO_EXTEND;
              Elt = DAG.getNode(Extend, dl, Ins[InsIdx].VT, Elt);
            }
            InVals.push_back(Elt);
          }
 
          // Reset vector tracking state.
          VecIdx = -1;
        }
        ++InsIdx;
      }
      if (VTs.size() > 0)
        --InsIdx;
      continue;
    }
 
    // Param has ByVal attribute
    // Return MoveParam(param symbol).
    // Ideally, the param symbol can be returned directly,
    // but when SDNode builder decides to use it in a CopyToReg(),
    // machine instruction fails because TargetExternalSymbol
    // (not lowered) is target dependent, and CopyToReg assumes
    // the source is lowered.
    EVT ObjectVT = getValueType(DL, Ty);
    assert(ObjectVT == Ins[InsIdx].VT &&
           "Ins type did not match function type");
    SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
    SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
    if (p.getNode())
      p.getNode()->setIROrder(idx + 1);
    InVals.push_back(p);
  }
 
  if (!OutChains.empty())
    DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains));
 
  return Chain;
}
 
SDValue
NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                                 bool isVarArg,
                                 const SmallVectorImpl<ISD::OutputArg> &Outs,
                                 const SmallVectorImpl<SDValue> &OutVals,
                                 const SDLoc &dl, SelectionDAG &DAG) const {
  const MachineFunction &MF = DAG.getMachineFunction();
  const Function &F = MF.getFunction();
  Type *RetTy = MF.getFunction().getReturnType();
 
  bool isABI = (STI.getSmVersion() >= 20);
  assert(isABI && "Non-ABI compilation is not supported");
  if (!isABI)
    return Chain;
 
  const DataLayout &DL = DAG.getDataLayout();
  SmallVector<SDValue, 16> PromotedOutVals;
  SmallVector<EVT, 16> VTs;
  SmallVector<uint64_t, 16> Offsets;
  ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets);
  assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
 
  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
    SDValue PromotedOutVal = OutVals[i];
    MVT PromotedVT;
    if (PromoteScalarIntegerPTX(VTs[i], &PromotedVT)) {
      VTs[i] = EVT(PromotedVT);
    }
    if (PromoteScalarIntegerPTX(PromotedOutVal.getValueType(), &PromotedVT)) {
      llvm::ISD::NodeType Ext =
          Outs[i].Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
      PromotedOutVal = DAG.getNode(Ext, dl, PromotedVT, PromotedOutVal);
    }
    PromotedOutVals.push_back(PromotedOutVal);
  }
 
  auto VectorInfo = VectorizePTXValueVTs(
      VTs, Offsets,
      RetTy->isSized() ? getFunctionParamOptimizedAlign(&F, RetTy, DL)
                       : Align(1));
 
  // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
  // 32-bits are sign extended or zero extended, depending on whether
  // they are signed or unsigned types.
  bool ExtendIntegerRetVal =
      RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
 
  SmallVector<SDValue, 6> StoreOperands;
  for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
    // New load/store. Record chain and offset operands.
    if (VectorInfo[i] & PVF_FIRST) {
      assert(StoreOperands.empty() && "Orphaned operand list.");
      StoreOperands.push_back(Chain);
      StoreOperands.push_back(DAG.getConstant(Offsets[i], dl, MVT::i32));
    }
 
    SDValue OutVal = OutVals[i];
    SDValue RetVal = PromotedOutVals[i];
 
    if (ExtendIntegerRetVal) {
      RetVal = DAG.getNode(Outs[i].Flags.isSExt() ? ISD::SIGN_EXTEND
                                                  : ISD::ZERO_EXTEND,
                           dl, MVT::i32, RetVal);
    } else if (OutVal.getValueSizeInBits() < 16) {
      // Use 16-bit registers for small load-stores as it's the
      // smallest general purpose register size supported by NVPTX.
      RetVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, RetVal);
    }
 
    // Record the value to return.
    StoreOperands.push_back(RetVal);
 
    // That's the last element of this store op.
    if (VectorInfo[i] & PVF_LAST) {
      NVPTXISD::NodeType Op;
      unsigned NumElts = StoreOperands.size() - 2;
      switch (NumElts) {
      case 1:
        Op = NVPTXISD::StoreRetval;
        break;
      case 2:
        Op = NVPTXISD::StoreRetvalV2;
        break;
      case 4:
        Op = NVPTXISD::StoreRetvalV4;
        break;
      default:
        llvm_unreachable("Invalid vector info.");
      }
 
      // Adjust type of load/store op if we've extended the scalar
      // return value.
      EVT TheStoreType = ExtendIntegerRetVal ? MVT::i32 : VTs[i];
      Chain = DAG.getMemIntrinsicNode(
          Op, dl, DAG.getVTList(MVT::Other), StoreOperands, TheStoreType,
          MachinePointerInfo(), Align(1), MachineMemOperand::MOStore);
      // Cleanup vector state.
      StoreOperands.clear();
    }
  }
 
  return DAG.getNode(NVPTXISD::RET_GLUE, dl, MVT::Other, Chain);
}
 
void NVPTXTargetLowering::LowerAsmOperandForConstraint(
    SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
    SelectionDAG &DAG) const {
  if (Constraint.length() > 1)
    return;
  else
    TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
}
 
static unsigned getOpcForTextureInstr(unsigned Intrinsic) {
  switch (Intrinsic) {
  default:
    return 0;
 
  case Intrinsic::nvvm_tex_1d_v4f32_s32:
    return NVPTXISD::Tex1DFloatS32;
  case Intrinsic::nvvm_tex_1d_v4f32_f32:
    return NVPTXISD::Tex1DFloatFloat;
  case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
    return NVPTXISD::Tex1DFloatFloatLevel;
  case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
    return NVPTXISD::Tex1DFloatFloatGrad;
  case Intrinsic::nvvm_tex_1d_v4s32_s32:
    return NVPTXISD::Tex1DS32S32;
  case Intrinsic::nvvm_tex_1d_v4s32_f32:
    return NVPTXISD::Tex1DS32Float;
  case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
    return NVPTXISD::Tex1DS32FloatLevel;
  case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
    return NVPTXISD::Tex1DS32FloatGrad;
  case Intrinsic::nvvm_tex_1d_v4u32_s32:
    return NVPTXISD::Tex1DU32S32;
  case Intrinsic::nvvm_tex_1d_v4u32_f32:
    return NVPTXISD::Tex1DU32Float;
  case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
    return NVPTXISD::Tex1DU32FloatLevel;
  case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
    return NVPTXISD::Tex1DU32FloatGrad;
 
  case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
    return NVPTXISD::Tex1DArrayFloatS32;
  case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
    return NVPTXISD::Tex1DArrayFloatFloat;
  case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
    return NVPTXISD::Tex1DArrayFloatFloatLevel;
  case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
    return NVPTXISD::Tex1DArrayFloatFloatGrad;
  case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
    return NVPTXISD::Tex1DArrayS32S32;
  case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
    return NVPTXISD::Tex1DArrayS32Float;
  case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
    return NVPTXISD::Tex1DArrayS32FloatLevel;
  case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
    return NVPTXISD::Tex1DArrayS32FloatGrad;
  case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
    return NVPTXISD::Tex1DArrayU32S32;
  case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
    return NVPTXISD::Tex1DArrayU32Float;
  case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
    return NVPTXISD::Tex1DArrayU32FloatLevel;
  case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
    return NVPTXISD::Tex1DArrayU32FloatGrad;
 
  case Intrinsic::nvvm_tex_2d_v4f32_s32:
    return NVPTXISD::Tex2DFloatS32;
  case Intrinsic::nvvm_tex_2d_v4f32_f32:
    return NVPTXISD::Tex2DFloatFloat;
  case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
    return NVPTXISD::Tex2DFloatFloatLevel;
  case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
    return NVPTXISD::Tex2DFloatFloatGrad;
  case Intrinsic::nvvm_tex_2d_v4s32_s32:
    return NVPTXISD::Tex2DS32S32;
  case Intrinsic::nvvm_tex_2d_v4s32_f32:
    return NVPTXISD::Tex2DS32Float;
  case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
    return NVPTXISD::Tex2DS32FloatLevel;
  case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
    return NVPTXISD::Tex2DS32FloatGrad;
  case Intrinsic::nvvm_tex_2d_v4u32_s32:
    return NVPTXISD::Tex2DU32S32;
  case Intrinsic::nvvm_tex_2d_v4u32_f32:
    return NVPTXISD::Tex2DU32Float;
  case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
    return NVPTXISD::Tex2DU32FloatLevel;
  case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
    return NVPTXISD::Tex2DU32FloatGrad;
 
  case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
    return NVPTXISD::Tex2DArrayFloatS32;
  case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
    return NVPTXISD::Tex2DArrayFloatFloat;
  case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
    return NVPTXISD::Tex2DArrayFloatFloatLevel;
  case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
    return NVPTXISD::Tex2DArrayFloatFloatGrad;
  case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
    return NVPTXISD::Tex2DArrayS32S32;
  case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
    return NVPTXISD::Tex2DArrayS32Float;
  case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
    return NVPTXISD::Tex2DArrayS32FloatLevel;
  case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
    return NVPTXISD::Tex2DArrayS32FloatGrad;
  case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
    return NVPTXISD::Tex2DArrayU32S32;
  case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
    return NVPTXISD::Tex2DArrayU32Float;
  case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
    return NVPTXISD::Tex2DArrayU32FloatLevel;
  case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
    return NVPTXISD::Tex2DArrayU32FloatGrad;
 
  case Intrinsic::nvvm_tex_3d_v4f32_s32:
    return NVPTXISD::Tex3DFloatS32;
  case Intrinsic::nvvm_tex_3d_v4f32_f32:
    return NVPTXISD::Tex3DFloatFloat;
  case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
    return NVPTXISD::Tex3DFloatFloatLevel;
  case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
    return NVPTXISD::Tex3DFloatFloatGrad;
  case Intrinsic::nvvm_tex_3d_v4s32_s32:
    return NVPTXISD::Tex3DS32S32;
  case Intrinsic::nvvm_tex_3d_v4s32_f32:
    return NVPTXISD::Tex3DS32Float;
  case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
    return NVPTXISD::Tex3DS32FloatLevel;
  case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
    return NVPTXISD::Tex3DS32FloatGrad;
  case Intrinsic::nvvm_tex_3d_v4u32_s32:
    return NVPTXISD::Tex3DU32S32;
  case Intrinsic::nvvm_tex_3d_v4u32_f32:
    return NVPTXISD::Tex3DU32Float;
  case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
    return NVPTXISD::Tex3DU32FloatLevel;
  case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
    return NVPTXISD::Tex3DU32FloatGrad;
 
  case Intrinsic::nvvm_tex_cube_v4f32_f32:
    return NVPTXISD::TexCubeFloatFloat;
  case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
    return NVPTXISD::TexCubeFloatFloatLevel;
  case Intrinsic::nvvm_tex_cube_v4s32_f32:
    return NVPTXISD::TexCubeS32Float;
  case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
    return NVPTXISD::TexCubeS32FloatLevel;
  case Intrinsic::nvvm_tex_cube_v4u32_f32:
    return NVPTXISD::TexCubeU32Float;
  case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
    return NVPTXISD::TexCubeU32FloatLevel;
 
  case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
    return NVPTXISD::TexCubeArrayFloatFloat;
  case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
    return NVPTXISD::TexCubeArrayFloatFloatLevel;
  case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
    return NVPTXISD::TexCubeArrayS32Float;
  case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
    return NVPTXISD::TexCubeArrayS32FloatLevel;
  case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
    return NVPTXISD::TexCubeArrayU32Float;
  case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
    return NVPTXISD::TexCubeArrayU32FloatLevel;
 
  case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
    return NVPTXISD::Tld4R2DFloatFloat;
  case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
    return NVPTXISD::Tld4G2DFloatFloat;
  case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
    return NVPTXISD::Tld4B2DFloatFloat;
  case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
    return NVPTXISD::Tld4A2DFloatFloat;
  case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
    return NVPTXISD::Tld4R2DS64Float;
  case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
    return NVPTXISD::Tld4G2DS64Float;
  case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
    return NVPTXISD::Tld4B2DS64Float;
  case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
    return NVPTXISD::Tld4A2DS64Float;
  case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
    return NVPTXISD::Tld4R2DU64Float;
  case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
    return NVPTXISD::Tld4G2DU64Float;
  case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
    return NVPTXISD::Tld4B2DU64Float;
  case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
    return NVPTXISD::Tld4A2DU64Float;
 
  case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
    return NVPTXISD::TexUnified1DFloatS32;
  case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
    return NVPTXISD::TexUnified1DFloatFloat;
  case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
    return NVPTXISD::TexUnified1DFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
    return NVPTXISD::TexUnified1DFloatFloatGrad;
  case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
    return NVPTXISD::TexUnified1DS32S32;
  case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
    return NVPTXISD::TexUnified1DS32Float;
  case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
    return NVPTXISD::TexUnified1DS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
    return NVPTXISD::TexUnified1DS32FloatGrad;
  case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
    return NVPTXISD::TexUnified1DU32S32;
  case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
    return NVPTXISD::TexUnified1DU32Float;
  case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
    return NVPTXISD::TexUnified1DU32FloatLevel;
  case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
    return NVPTXISD::TexUnified1DU32FloatGrad;
 
  case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
    return NVPTXISD::TexUnified1DArrayFloatS32;
  case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
    return NVPTXISD::TexUnified1DArrayFloatFloat;
  case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
    return NVPTXISD::TexUnified1DArrayFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
    return NVPTXISD::TexUnified1DArrayFloatFloatGrad;
  case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
    return NVPTXISD::TexUnified1DArrayS32S32;
  case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
    return NVPTXISD::TexUnified1DArrayS32Float;
  case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
    return NVPTXISD::TexUnified1DArrayS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
    return NVPTXISD::TexUnified1DArrayS32FloatGrad;
  case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
    return NVPTXISD::TexUnified1DArrayU32S32;
  case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
    return NVPTXISD::TexUnified1DArrayU32Float;
  case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
    return NVPTXISD::TexUnified1DArrayU32FloatLevel;
  case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
    return NVPTXISD::TexUnified1DArrayU32FloatGrad;
 
  case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
    return NVPTXISD::TexUnified2DFloatS32;
  case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
    return NVPTXISD::TexUnified2DFloatFloat;
  case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
    return NVPTXISD::TexUnified2DFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
    return NVPTXISD::TexUnified2DFloatFloatGrad;
  case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
    return NVPTXISD::TexUnified2DS32S32;
  case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
    return NVPTXISD::TexUnified2DS32Float;
  case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
    return NVPTXISD::TexUnified2DS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
    return NVPTXISD::TexUnified2DS32FloatGrad;
  case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
    return NVPTXISD::TexUnified2DU32S32;
  case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
    return NVPTXISD::TexUnified2DU32Float;
  case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
    return NVPTXISD::TexUnified2DU32FloatLevel;
  case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
    return NVPTXISD::TexUnified2DU32FloatGrad;
 
  case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
    return NVPTXISD::TexUnified2DArrayFloatS32;
  case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
    return NVPTXISD::TexUnified2DArrayFloatFloat;
  case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
    return NVPTXISD::TexUnified2DArrayFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
    return NVPTXISD::TexUnified2DArrayFloatFloatGrad;
  case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
    return NVPTXISD::TexUnified2DArrayS32S32;
  case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
    return NVPTXISD::TexUnified2DArrayS32Float;
  case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
    return NVPTXISD::TexUnified2DArrayS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
    return NVPTXISD::TexUnified2DArrayS32FloatGrad;
  case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
    return NVPTXISD::TexUnified2DArrayU32S32;
  case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
    return NVPTXISD::TexUnified2DArrayU32Float;
  case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
    return NVPTXISD::TexUnified2DArrayU32FloatLevel;
  case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
    return NVPTXISD::TexUnified2DArrayU32FloatGrad;
 
  case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
    return NVPTXISD::TexUnified3DFloatS32;
  case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
    return NVPTXISD::TexUnified3DFloatFloat;
  case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
    return NVPTXISD::TexUnified3DFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
    return NVPTXISD::TexUnified3DFloatFloatGrad;
  case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
    return NVPTXISD::TexUnified3DS32S32;
  case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
    return NVPTXISD::TexUnified3DS32Float;
  case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
    return NVPTXISD::TexUnified3DS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
    return NVPTXISD::TexUnified3DS32FloatGrad;
  case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
    return NVPTXISD::TexUnified3DU32S32;
  case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
    return NVPTXISD::TexUnified3DU32Float;
  case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
    return NVPTXISD::TexUnified3DU32FloatLevel;
  case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
    return NVPTXISD::TexUnified3DU32FloatGrad;
 
  case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
    return NVPTXISD::TexUnifiedCubeFloatFloat;
  case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
    return NVPTXISD::TexUnifiedCubeFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
    return NVPTXISD::TexUnifiedCubeS32Float;
  case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
    return NVPTXISD::TexUnifiedCubeS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
    return NVPTXISD::TexUnifiedCubeU32Float;
  case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
    return NVPTXISD::TexUnifiedCubeU32FloatLevel;
 
  case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
    return NVPTXISD::TexUnifiedCubeArrayFloatFloat;
  case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
    return NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel;
  case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
    return NVPTXISD::TexUnifiedCubeArrayS32Float;
  case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
    return NVPTXISD::TexUnifiedCubeArrayS32FloatLevel;
  case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
    return NVPTXISD::TexUnifiedCubeArrayU32Float;
  case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
    return NVPTXISD::TexUnifiedCubeArrayU32FloatLevel;
 
  case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
    return NVPTXISD::Tld4UnifiedR2DFloatFloat;
  case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
    return NVPTXISD::Tld4UnifiedG2DFloatFloat;
  case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
    return NVPTXISD::Tld4UnifiedB2DFloatFloat;
  case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
    return NVPTXISD::Tld4UnifiedA2DFloatFloat;
  case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
    return NVPTXISD::Tld4UnifiedR2DS64Float;
  case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
    return NVPTXISD::Tld4UnifiedG2DS64Float;
  case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
    return NVPTXISD::Tld4UnifiedB2DS64Float;
  case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
    return NVPTXISD::Tld4UnifiedA2DS64Float;
  case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
    return NVPTXISD::Tld4UnifiedR2DU64Float;
  case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
    return NVPTXISD::Tld4UnifiedG2DU64Float;
  case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
    return NVPTXISD::Tld4UnifiedB2DU64Float;
  case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
    return NVPTXISD::Tld4UnifiedA2DU64Float;
  }
}
 
static unsigned getOpcForSurfaceInstr(unsigned Intrinsic) {
  switch (Intrinsic) {
  default:
    return 0;
  case Intrinsic::nvvm_suld_1d_i8_clamp:
    return NVPTXISD::Suld1DI8Clamp;
  case Intrinsic::nvvm_suld_1d_i16_clamp:
    return NVPTXISD::Suld1DI16Clamp;
  case Intrinsic::nvvm_suld_1d_i32_clamp:
    return NVPTXISD::Suld1DI32Clamp;
  case Intrinsic::nvvm_suld_1d_i64_clamp:
    return NVPTXISD::Suld1DI64Clamp;
  case Intrinsic::nvvm_suld_1d_v2i8_clamp:
    return NVPTXISD::Suld1DV2I8Clamp;
  case Intrinsic::nvvm_suld_1d_v2i16_clamp:
    return NVPTXISD::Suld1DV2I16Clamp;
  case Intrinsic::nvvm_suld_1d_v2i32_clamp:
    return NVPTXISD::Suld1DV2I32Clamp;
  case Intrinsic::nvvm_suld_1d_v2i64_clamp:
    return NVPTXISD::Suld1DV2I64Clamp;
  case Intrinsic::nvvm_suld_1d_v4i8_clamp:
    return NVPTXISD::Suld1DV4I8Clamp;
  case Intrinsic::nvvm_suld_1d_v4i16_clamp:
    return NVPTXISD::Suld1DV4I16Clamp;
  case Intrinsic::nvvm_suld_1d_v4i32_clamp:
    return NVPTXISD::Suld1DV4I32Clamp;
  case Intrinsic::nvvm_suld_1d_array_i8_clamp:
    return NVPTXISD::Suld1DArrayI8Clamp;
  case Intrinsic::nvvm_suld_1d_array_i16_clamp:
    return NVPTXISD::Suld1DArrayI16Clamp;
  case Intrinsic::nvvm_suld_1d_array_i32_clamp:
    return NVPTXISD::Suld1DArrayI32Clamp;
  case Intrinsic::nvvm_suld_1d_array_i64_clamp:
    return NVPTXISD::Suld1DArrayI64Clamp;
  case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
    return NVPTXISD::Suld1DArrayV2I8Clamp;
  case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
    return NVPTXISD::Suld1DArrayV2I16Clamp;
  case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
    return NVPTXISD::Suld1DArrayV2I32Clamp;
  case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
    return NVPTXISD::Suld1DArrayV2I64Clamp;
  case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
    return NVPTXISD::Suld1DArrayV4I8Clamp;
  case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
    return NVPTXISD::Suld1DArrayV4I16Clamp;
  case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
    return NVPTXISD::Suld1DArrayV4I32Clamp;
  case Intrinsic::nvvm_suld_2d_i8_clamp:
    return NVPTXISD::Suld2DI8Clamp;
  case Intrinsic::nvvm_suld_2d_i16_clamp:
    return NVPTXISD::Suld2DI16Clamp;
  case Intrinsic::nvvm_suld_2d_i32_clamp:
    return NVPTXISD::Suld2DI32Clamp;
  case Intrinsic::nvvm_suld_2d_i64_clamp:
    return NVPTXISD::Suld2DI64Clamp;
  case Intrinsic::nvvm_suld_2d_v2i8_clamp:
    return NVPTXISD::Suld2DV2I8Clamp;
  case Intrinsic::nvvm_suld_2d_v2i16_clamp:
    return NVPTXISD::Suld2DV2I16Clamp;
  case Intrinsic::nvvm_suld_2d_v2i32_clamp:
    return NVPTXISD::Suld2DV2I32Clamp;
  case Intrinsic::nvvm_suld_2d_v2i64_clamp:
    return NVPTXISD::Suld2DV2I64Clamp;
  case Intrinsic::nvvm_suld_2d_v4i8_clamp:
    return NVPTXISD::Suld2DV4I8Clamp;
  case Intrinsic::nvvm_suld_2d_v4i16_clamp:
    return NVPTXISD::Suld2DV4I16Clamp;
  case Intrinsic::nvvm_suld_2d_v4i32_clamp:
    return NVPTXISD::Suld2DV4I32Clamp;
  case Intrinsic::nvvm_suld_2d_array_i8_clamp:
    return NVPTXISD::Suld2DArrayI8Clamp;
  case Intrinsic::nvvm_suld_2d_array_i16_clamp:
    return NVPTXISD::Suld2DArrayI16Clamp;
  case Intrinsic::nvvm_suld_2d_array_i32_clamp:
    return NVPTXISD::Suld2DArrayI32Clamp;
  case Intrinsic::nvvm_suld_2d_array_i64_clamp:
    return NVPTXISD::Suld2DArrayI64Clamp;
  case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
    return NVPTXISD::Suld2DArrayV2I8Clamp;
  case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
    return NVPTXISD::Suld2DArrayV2I16Clamp;
  case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
    return NVPTXISD::Suld2DArrayV2I32Clamp;
  case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
    return NVPTXISD::Suld2DArrayV2I64Clamp;
  case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
    return NVPTXISD::Suld2DArrayV4I8Clamp;
  case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
    return NVPTXISD::Suld2DArrayV4I16Clamp;
  case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
    return NVPTXISD::Suld2DArrayV4I32Clamp;
  case Intrinsic::nvvm_suld_3d_i8_clamp:
    return NVPTXISD::Suld3DI8Clamp;
  case Intrinsic::nvvm_suld_3d_i16_clamp:
    return NVPTXISD::Suld3DI16Clamp;
  case Intrinsic::nvvm_suld_3d_i32_clamp:
    return NVPTXISD::Suld3DI32Clamp;
  case Intrinsic::nvvm_suld_3d_i64_clamp:
    return NVPTXISD::Suld3DI64Clamp;
  case Intrinsic::nvvm_suld_3d_v2i8_clamp:
    return NVPTXISD::Suld3DV2I8Clamp;
  case Intrinsic::nvvm_suld_3d_v2i16_clamp:
    return NVPTXISD::Suld3DV2I16Clamp;
  case Intrinsic::nvvm_suld_3d_v2i32_clamp:
    return NVPTXISD::Suld3DV2I32Clamp;
  case Intrinsic::nvvm_suld_3d_v2i64_clamp:
    return NVPTXISD::Suld3DV2I64Clamp;
  case Intrinsic::nvvm_suld_3d_v4i8_clamp:
    return NVPTXISD::Suld3DV4I8Clamp;
  case Intrinsic::nvvm_suld_3d_v4i16_clamp:
    return NVPTXISD::Suld3DV4I16Clamp;
  case Intrinsic::nvvm_suld_3d_v4i32_clamp:
    return NVPTXISD::Suld3DV4I32Clamp;
  case Intrinsic::nvvm_suld_1d_i8_trap:
    return NVPTXISD::Suld1DI8Trap;
  case Intrinsic::nvvm_suld_1d_i16_trap:
    return NVPTXISD::Suld1DI16Trap;
  case Intrinsic::nvvm_suld_1d_i32_trap:
    return NVPTXISD::Suld1DI32Trap;
  case Intrinsic::nvvm_suld_1d_i64_trap:
    return NVPTXISD::Suld1DI64Trap;
  case Intrinsic::nvvm_suld_1d_v2i8_trap:
    return NVPTXISD::Suld1DV2I8Trap;
  case Intrinsic::nvvm_suld_1d_v2i16_trap:
    return NVPTXISD::Suld1DV2I16Trap;
  case Intrinsic::nvvm_suld_1d_v2i32_trap:
    return NVPTXISD::Suld1DV2I32Trap;
  case Intrinsic::nvvm_suld_1d_v2i64_trap:
    return NVPTXISD::Suld1DV2I64Trap;
  case Intrinsic::nvvm_suld_1d_v4i8_trap:
    return NVPTXISD::Suld1DV4I8Trap;
  case Intrinsic::nvvm_suld_1d_v4i16_trap:
    return NVPTXISD::Suld1DV4I16Trap;
  case Intrinsic::nvvm_suld_1d_v4i32_trap:
    return NVPTXISD::Suld1DV4I32Trap;
  case Intrinsic::nvvm_suld_1d_array_i8_trap:
    return NVPTXISD::Suld1DArrayI8Trap;
  case Intrinsic::nvvm_suld_1d_array_i16_trap:
    return NVPTXISD::Suld1DArrayI16Trap;
  case Intrinsic::nvvm_suld_1d_array_i32_trap:
    return NVPTXISD::Suld1DArrayI32Trap;
  case Intrinsic::nvvm_suld_1d_array_i64_trap:
    return NVPTXISD::Suld1DArrayI64Trap;
  case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
    return NVPTXISD::Suld1DArrayV2I8Trap;
  case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
    return NVPTXISD::Suld1DArrayV2I16Trap;
  case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
    return NVPTXISD::Suld1DArrayV2I32Trap;
  case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
    return NVPTXISD::Suld1DArrayV2I64Trap;
  case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
    return NVPTXISD::Suld1DArrayV4I8Trap;
  case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
    return NVPTXISD::Suld1DArrayV4I16Trap;
  case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
    return NVPTXISD::Suld1DArrayV4I32Trap;
  case Intrinsic::nvvm_suld_2d_i8_trap:
    return NVPTXISD::Suld2DI8Trap;
  case Intrinsic::nvvm_suld_2d_i16_trap:
    return NVPTXISD::Suld2DI16Trap;
  case Intrinsic::nvvm_suld_2d_i32_trap:
    return NVPTXISD::Suld2DI32Trap;
  case Intrinsic::nvvm_suld_2d_i64_trap:
    return NVPTXISD::Suld2DI64Trap;
  case Intrinsic::nvvm_suld_2d_v2i8_trap:
    return NVPTXISD::Suld2DV2I8Trap;
  case Intrinsic::nvvm_suld_2d_v2i16_trap:
    return NVPTXISD::Suld2DV2I16Trap;
  case Intrinsic::nvvm_suld_2d_v2i32_trap:
    return NVPTXISD::Suld2DV2I32Trap;
  case Intrinsic::nvvm_suld_2d_v2i64_trap:
    return NVPTXISD::Suld2DV2I64Trap;
  case Intrinsic::nvvm_suld_2d_v4i8_trap:
    return NVPTXISD::Suld2DV4I8Trap;
  case Intrinsic::nvvm_suld_2d_v4i16_trap:
    return NVPTXISD::Suld2DV4I16Trap;
  case Intrinsic::nvvm_suld_2d_v4i32_trap:
    return NVPTXISD::Suld2DV4I32Trap;
  case Intrinsic::nvvm_suld_2d_array_i8_trap:
    return NVPTXISD::Suld2DArrayI8Trap;
  case Intrinsic::nvvm_suld_2d_array_i16_trap:
    return NVPTXISD::Suld2DArrayI16Trap;
  case Intrinsic::nvvm_suld_2d_array_i32_trap:
    return NVPTXISD::Suld2DArrayI32Trap;
  case Intrinsic::nvvm_suld_2d_array_i64_trap:
    return NVPTXISD::Suld2DArrayI64Trap;
  case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
    return NVPTXISD::Suld2DArrayV2I8Trap;
  case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
    return NVPTXISD::Suld2DArrayV2I16Trap;
  case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
    return NVPTXISD::Suld2DArrayV2I32Trap;
  case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
    return NVPTXISD::Suld2DArrayV2I64Trap;
  case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
    return NVPTXISD::Suld2DArrayV4I8Trap;
  case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
    return NVPTXISD::Suld2DArrayV4I16Trap;
  case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
    return NVPTXISD::Suld2DArrayV4I32Trap;
  case Intrinsic::nvvm_suld_3d_i8_trap:
    return NVPTXISD::Suld3DI8Trap;
  case Intrinsic::nvvm_suld_3d_i16_trap:
    return NVPTXISD::Suld3DI16Trap;
  case Intrinsic::nvvm_suld_3d_i32_trap:
    return NVPTXISD::Suld3DI32Trap;
  case Intrinsic::nvvm_suld_3d_i64_trap:
    return NVPTXISD::Suld3DI64Trap;
  case Intrinsic::nvvm_suld_3d_v2i8_trap:
    return NVPTXISD::Suld3DV2I8Trap;
  case Intrinsic::nvvm_suld_3d_v2i16_trap:
    return NVPTXISD::Suld3DV2I16Trap;
  case Intrinsic::nvvm_suld_3d_v2i32_trap:
    return NVPTXISD::Suld3DV2I32Trap;
  case Intrinsic::nvvm_suld_3d_v2i64_trap:
    return NVPTXISD::Suld3DV2I64Trap;
  case Intrinsic::nvvm_suld_3d_v4i8_trap:
    return NVPTXISD::Suld3DV4I8Trap;
  case Intrinsic::nvvm_suld_3d_v4i16_trap:
    return NVPTXISD::Suld3DV4I16Trap;
  case Intrinsic::nvvm_suld_3d_v4i32_trap:
    return NVPTXISD::Suld3DV4I32Trap;
  case Intrinsic::nvvm_suld_1d_i8_zero:
    return NVPTXISD::Suld1DI8Zero;
  case Intrinsic::nvvm_suld_1d_i16_zero:
    return NVPTXISD::Suld1DI16Zero;
  case Intrinsic::nvvm_suld_1d_i32_zero:
    return NVPTXISD::Suld1DI32Zero;
  case Intrinsic::nvvm_suld_1d_i64_zero:
    return NVPTXISD::Suld1DI64Zero;
  case Intrinsic::nvvm_suld_1d_v2i8_zero:
    return NVPTXISD::Suld1DV2I8Zero;
  case Intrinsic::nvvm_suld_1d_v2i16_zero:
    return NVPTXISD::Suld1DV2I16Zero;
  case Intrinsic::nvvm_suld_1d_v2i32_zero:
    return NVPTXISD::Suld1DV2I32Zero;
  case Intrinsic::nvvm_suld_1d_v2i64_zero:
    return NVPTXISD::Suld1DV2I64Zero;
  case Intrinsic::nvvm_suld_1d_v4i8_zero:
    return NVPTXISD::Suld1DV4I8Zero;
  case Intrinsic::nvvm_suld_1d_v4i16_zero:
    return NVPTXISD::Suld1DV4I16Zero;
  case Intrinsic::nvvm_suld_1d_v4i32_zero:
    return NVPTXISD::Suld1DV4I32Zero;
  case Intrinsic::nvvm_suld_1d_array_i8_zero:
    return NVPTXISD::Suld1DArrayI8Zero;
  case Intrinsic::nvvm_suld_1d_array_i16_zero:
    return NVPTXISD::Suld1DArrayI16Zero;
  case Intrinsic::nvvm_suld_1d_array_i32_zero:
    return NVPTXISD::Suld1DArrayI32Zero;
  case Intrinsic::nvvm_suld_1d_array_i64_zero:
    return NVPTXISD::Suld1DArrayI64Zero;
  case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
    return NVPTXISD::Suld1DArrayV2I8Zero;
  case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
    return NVPTXISD::Suld1DArrayV2I16Zero;
  case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
    return NVPTXISD::Suld1DArrayV2I32Zero;
  case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
    return NVPTXISD::Suld1DArrayV2I64Zero;
  case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
    return NVPTXISD::Suld1DArrayV4I8Zero;
  case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
    return NVPTXISD::Suld1DArrayV4I16Zero;
  case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
    return NVPTXISD::Suld1DArrayV4I32Zero;
  case Intrinsic::nvvm_suld_2d_i8_zero:
    return NVPTXISD::Suld2DI8Zero;
  case Intrinsic::nvvm_suld_2d_i16_zero:
    return NVPTXISD::Suld2DI16Zero;
  case Intrinsic::nvvm_suld_2d_i32_zero:
    return NVPTXISD::Suld2DI32Zero;
  case Intrinsic::nvvm_suld_2d_i64_zero:
    return NVPTXISD::Suld2DI64Zero;
  case Intrinsic::nvvm_suld_2d_v2i8_zero:
    return NVPTXISD::Suld2DV2I8Zero;
  case Intrinsic::nvvm_suld_2d_v2i16_zero:
    return NVPTXISD::Suld2DV2I16Zero;
  case Intrinsic::nvvm_suld_2d_v2i32_zero:
    return NVPTXISD::Suld2DV2I32Zero;
  case Intrinsic::nvvm_suld_2d_v2i64_zero:
    return NVPTXISD::Suld2DV2I64Zero;
  case Intrinsic::nvvm_suld_2d_v4i8_zero:
    return NVPTXISD::Suld2DV4I8Zero;
  case Intrinsic::nvvm_suld_2d_v4i16_zero:
    return NVPTXISD::Suld2DV4I16Zero;
  case Intrinsic::nvvm_suld_2d_v4i32_zero:
    return NVPTXISD::Suld2DV4I32Zero;
  case Intrinsic::nvvm_suld_2d_array_i8_zero:
    return NVPTXISD::Suld2DArrayI8Zero;
  case Intrinsic::nvvm_suld_2d_array_i16_zero:
    return NVPTXISD::Suld2DArrayI16Zero;
  case Intrinsic::nvvm_suld_2d_array_i32_zero:
    return NVPTXISD::Suld2DArrayI32Zero;
  case Intrinsic::nvvm_suld_2d_array_i64_zero:
    return NVPTXISD::Suld2DArrayI64Zero;
  case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
    return NVPTXISD::Suld2DArrayV2I8Zero;
  case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
    return NVPTXISD::Suld2DArrayV2I16Zero;
  case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
    return NVPTXISD::Suld2DArrayV2I32Zero;
  case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
    return NVPTXISD::Suld2DArrayV2I64Zero;
  case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
    return NVPTXISD::Suld2DArrayV4I8Zero;
  case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
    return NVPTXISD::Suld2DArrayV4I16Zero;
  case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
    return NVPTXISD::Suld2DArrayV4I32Zero;
  case Intrinsic::nvvm_suld_3d_i8_zero:
    return NVPTXISD::Suld3DI8Zero;
  case Intrinsic::nvvm_suld_3d_i16_zero:
    return NVPTXISD::Suld3DI16Zero;
  case Intrinsic::nvvm_suld_3d_i32_zero:
    return NVPTXISD::Suld3DI32Zero;
  case Intrinsic::nvvm_suld_3d_i64_zero:
    return NVPTXISD::Suld3DI64Zero;
  case Intrinsic::nvvm_suld_3d_v2i8_zero:
    return NVPTXISD::Suld3DV2I8Zero;
  case Intrinsic::nvvm_suld_3d_v2i16_zero:
    return NVPTXISD::Suld3DV2I16Zero;
  case Intrinsic::nvvm_suld_3d_v2i32_zero:
    return NVPTXISD::Suld3DV2I32Zero;
  case Intrinsic::nvvm_suld_3d_v2i64_zero:
    return NVPTXISD::Suld3DV2I64Zero;
  case Intrinsic::nvvm_suld_3d_v4i8_zero:
    return NVPTXISD::Suld3DV4I8Zero;
  case Intrinsic::nvvm_suld_3d_v4i16_zero:
    return NVPTXISD::Suld3DV4I16Zero;
  case Intrinsic::nvvm_suld_3d_v4i32_zero:
    return NVPTXISD::Suld3DV4I32Zero;
  }
}
 
// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
// TgtMemIntrinsic
// because we need the information that is only available in the "Value" type
// of destination
// pointer. In particular, the address space information.
bool NVPTXTargetLowering::getTgtMemIntrinsic(
    IntrinsicInfo &Info, const CallInst &I,
    MachineFunction &MF, unsigned Intrinsic) const {
  switch (Intrinsic) {
  default:
    return false;
  case Intrinsic::nvvm_match_all_sync_i32p:
  case Intrinsic::nvvm_match_all_sync_i64p:
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute
    // in order to model data exchange with other threads, but perform no real
    // memory accesses.
    Info.memVT = MVT::i1;
 
    // Our result depends on both our and other thread's arguments.
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    return true;
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v8f16;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(16);
    return true;
  }
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v2i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(8);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
  case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
  case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
 
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
  case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
  case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
  case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
  case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v4i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
 
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
  case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
  case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
  case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
  case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
  case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
  case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
  case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
  case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
  case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(4);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v4f16;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
  case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
  case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v8f32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
 
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
 
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
  case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
  case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
  case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v8i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
  case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
  case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
  case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
  case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
  case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
  case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v2i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(8);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
  case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
  case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
 
  case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
  case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
  case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::f64;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(8);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
  case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
  case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v2f64;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = MVT::v4f16;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOStore;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
  case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
  case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = MVT::v8f32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOStore;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
  case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
  case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
  case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = MVT::v8i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOStore;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
  case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
  case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
  case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
  case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = MVT::v2i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOStore;
    Info.align = Align(8);
    return true;
  }
 
  case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
  case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
  case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
  case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = MVT::v2f64;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOStore;
    Info.align = Align(16);
    return true;
  }
 
  case Intrinsic::nvvm_atomic_load_inc_32:
  case Intrinsic::nvvm_atomic_load_dec_32:
 
  case Intrinsic::nvvm_atomic_add_gen_f_cta:
  case Intrinsic::nvvm_atomic_add_gen_f_sys:
  case Intrinsic::nvvm_atomic_add_gen_i_cta:
  case Intrinsic::nvvm_atomic_add_gen_i_sys:
  case Intrinsic::nvvm_atomic_and_gen_i_cta:
  case Intrinsic::nvvm_atomic_and_gen_i_sys:
  case Intrinsic::nvvm_atomic_cas_gen_i_cta:
  case Intrinsic::nvvm_atomic_cas_gen_i_sys:
  case Intrinsic::nvvm_atomic_dec_gen_i_cta:
  case Intrinsic::nvvm_atomic_dec_gen_i_sys:
  case Intrinsic::nvvm_atomic_inc_gen_i_cta:
  case Intrinsic::nvvm_atomic_inc_gen_i_sys:
  case Intrinsic::nvvm_atomic_max_gen_i_cta:
  case Intrinsic::nvvm_atomic_max_gen_i_sys:
  case Intrinsic::nvvm_atomic_min_gen_i_cta:
  case Intrinsic::nvvm_atomic_min_gen_i_sys:
  case Intrinsic::nvvm_atomic_or_gen_i_cta:
  case Intrinsic::nvvm_atomic_or_gen_i_sys:
  case Intrinsic::nvvm_atomic_exch_gen_i_cta:
  case Intrinsic::nvvm_atomic_exch_gen_i_sys:
  case Intrinsic::nvvm_atomic_xor_gen_i_cta:
  case Intrinsic::nvvm_atomic_xor_gen_i_sys: {
    auto &DL = I.getModule()->getDataLayout();
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = getValueType(DL, I.getType());
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    Info.align.reset();
    return true;
  }
 
  case Intrinsic::nvvm_ldu_global_i:
  case Intrinsic::nvvm_ldu_global_f:
  case Intrinsic::nvvm_ldu_global_p: {
    auto &DL = I.getModule()->getDataLayout();
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
      Info.memVT = getValueType(DL, I.getType());
    else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)
      Info.memVT = getPointerTy(DL);
    else
      Info.memVT = getValueType(DL, I.getType());
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
 
    return true;
  }
  case Intrinsic::nvvm_ldg_global_i:
  case Intrinsic::nvvm_ldg_global_f:
  case Intrinsic::nvvm_ldg_global_p: {
    auto &DL = I.getModule()->getDataLayout();
 
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
      Info.memVT = getValueType(DL, I.getType());
    else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
      Info.memVT = getPointerTy(DL);
    else
      Info.memVT = getValueType(DL, I.getType());
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.flags = MachineMemOperand::MOLoad;
    Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
 
    return true;
  }
 
  case Intrinsic::nvvm_tex_1d_v4f32_s32:
  case Intrinsic::nvvm_tex_1d_v4f32_f32:
  case Intrinsic::nvvm_tex_1d_level_v4f32_f32: