NVVMProperties.cpp

Go to the documentation of this file.

//===- NVVMProperties.cpp - NVVM annotation utilities ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains NVVM attribute and metadata query utilities.
//
//===----------------------------------------------------------------------===//
 
#include "NVVMProperties.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/NVVMAttributes.h"
#include <functional>
#include <map>
#include <mutex>
#include <numeric>
#include <string>
#include <vector>
 
using namespace llvm;
 
namespace {
using AnnotationValues = std::map<std::string, std::vector<unsigned>>;
using AnnotationMap = std::map<const GlobalValue *, AnnotationValues>;
 
struct AnnotationCache {
  sys::Mutex Lock;
  std::map<const Module *, AnnotationMap> Cache;
};
 
} // namespace
 
static AnnotationCache &getAnnotationCache() {
  static AnnotationCache AC;
  return AC;
}
 
// TODO: Replace these legacy nvvm.annotations metadata names with proper
// function/parameter attributes (like the NVVMAttr:: constants).
namespace NVVMMetadata {
constexpr StringLiteral Texture("texture");
constexpr StringLiteral Surface("surface");
constexpr StringLiteral Sampler("sampler");
constexpr StringLiteral ReadOnlyImage("rdoimage");
constexpr StringLiteral WriteOnlyImage("wroimage");
constexpr StringLiteral ReadWriteImage("rdwrimage");
constexpr StringLiteral Managed("managed");
} // namespace NVVMMetadata
 
void llvm::clearAnnotationCache(const Module *Mod) {
  auto &AC = getAnnotationCache();
  std::lock_guard<sys::Mutex> Guard(AC.Lock);
  AC.Cache.erase(Mod);
}
 
static void cacheAnnotationFromMD(const MDNode *MetadataNode,
                                  AnnotationValues &RetVal) {
  auto &AC = getAnnotationCache();
  std::lock_guard<sys::Mutex> Guard(AC.Lock);
  assert(MetadataNode && "Invalid mdnode for annotation");
  assert((MetadataNode->getNumOperands() % 2) == 1 &&
         "Invalid number of operands");
  // start index = 1, to skip the global variable key
  // increment = 2, to skip the value for each property-value pairs
  for (unsigned I = 1, E = MetadataNode->getNumOperands(); I != E; I += 2) {
    const MDString *Prop = dyn_cast<MDString>(MetadataNode->getOperand(I));
    assert(Prop && "Annotation property not a string");
    std::string Key = Prop->getString().str();
 
    if (ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(
            MetadataNode->getOperand(I + 1))) {
      RetVal[Key].push_back(Val->getZExtValue());
    } else {
      llvm_unreachable("Value operand not a constant int");
    }
  }
}
 
static void cacheAnnotationFromMD(const Module *M, const GlobalValue *GV) {
  auto &AC = getAnnotationCache();
  std::lock_guard<sys::Mutex> Guard(AC.Lock);
  NamedMDNode *NMD = M->getNamedMetadata("nvvm.annotations");
  if (!NMD)
    return;
 
  AnnotationValues Tmp;
  for (unsigned I = 0, E = NMD->getNumOperands(); I != E; ++I) {
    const MDNode *Elem = NMD->getOperand(I);
    GlobalValue *Entity =
        mdconst::dyn_extract_or_null<GlobalValue>(Elem->getOperand(0));
    // entity may be null due to DCE
    if (!Entity || Entity != GV)
      continue;
 
    cacheAnnotationFromMD(Elem, Tmp);
  }
 
  if (Tmp.empty())
    return;
 
  AC.Cache[M][GV] = std::move(Tmp);
}
 
static std::optional<unsigned> findOneNVVMAnnotation(const GlobalValue *GV,
                                                     StringRef Prop) {
  auto &AC = getAnnotationCache();
  std::lock_guard<sys::Mutex> Guard(AC.Lock);
  const Module *M = GV->getParent();
  auto ACIt = AC.Cache.find(M);
  if (ACIt == AC.Cache.end())
    cacheAnnotationFromMD(M, GV);
  else if (ACIt->second.find(GV) == ACIt->second.end())
    cacheAnnotationFromMD(M, GV);
 
  auto &KVP = AC.Cache[M][GV];
  auto It = KVP.find(Prop.str());
  if (It == KVP.end())
    return std::nullopt;
  return It->second[0];
}
 
static bool findAllNVVMAnnotation(const GlobalValue *GV, StringRef Prop,
                                  std::vector<unsigned> &RetVal) {
  auto &AC = getAnnotationCache();
  std::lock_guard<sys::Mutex> Guard(AC.Lock);
  const Module *M = GV->getParent();
  auto ACIt = AC.Cache.find(M);
  if (ACIt == AC.Cache.end())
    cacheAnnotationFromMD(M, GV);
  else if (ACIt->second.find(GV) == ACIt->second.end())
    cacheAnnotationFromMD(M, GV);
 
  auto &KVP = AC.Cache[M][GV];
  auto It = KVP.find(Prop.str());
  if (It == KVP.end())
    return false;
  RetVal = It->second;
  return true;
}
 
static bool globalHasNVVMAnnotation(const Value &V, StringRef Prop) {
  if (const auto *GV = dyn_cast<GlobalValue>(&V))
    if (const auto Annot = findOneNVVMAnnotation(GV, Prop)) {
      assert((*Annot == 1) && "Unexpected annotation on a symbol");
      return true;
    }
 
  return false;
}
 
static bool argHasNVVMAnnotation(const Value &Val, StringRef Annotation) {
  if (const auto *Arg = dyn_cast<Argument>(&Val)) {
    std::vector<unsigned> Annot;
    if (findAllNVVMAnnotation(Arg->getParent(), Annotation, Annot) &&
        is_contained(Annot, Arg->getArgNo()))
      return true;
  }
  return false;
}
 
static std::optional<unsigned> getFnAttrParsedInt(const Function &F,
                                                  StringRef Attr) {
  return F.hasFnAttribute(Attr)
             ? std::optional(F.getFnAttributeAsParsedInteger(Attr))
             : std::nullopt;
}
 
static SmallVector<unsigned, 3> getFnAttrParsedVector(const Function &F,
                                                      StringRef Attr) {
  SmallVector<unsigned, 3> V;
  auto &Ctx = F.getContext();
 
  if (F.hasFnAttribute(Attr)) {
    // We expect the attribute value to be of the form "x[,y[,z]]", where x, y,
    // and z are unsigned values.
    StringRef S = F.getFnAttribute(Attr).getValueAsString();
    for (unsigned I = 0; I < 3 && !S.empty(); I++) {
      auto [First, Rest] = S.split(",");
      unsigned IntVal;
      if (First.trim().getAsInteger(0, IntVal))
        Ctx.emitError("can't parse integer attribute " + First + " in " + Attr);
 
      V.push_back(IntVal);
      S = Rest;
    }
  }
  return V;
}
 
static std::optional<uint64_t> getVectorProduct(ArrayRef<unsigned> V) {
  if (V.empty())
    return std::nullopt;
 
  return std::accumulate(V.begin(), V.end(), uint64_t(1),
                         std::multiplies<uint64_t>{});
}
 
PTXOpaqueType llvm::getPTXOpaqueType(const GlobalVariable &GV) {
  if (findOneNVVMAnnotation(&GV, NVVMMetadata::Texture))
    return PTXOpaqueType::Texture;
  if (findOneNVVMAnnotation(&GV, NVVMMetadata::Surface))
    return PTXOpaqueType::Surface;
  if (findOneNVVMAnnotation(&GV, NVVMMetadata::Sampler))
    return PTXOpaqueType::Sampler;
  return PTXOpaqueType::None;
}
 
PTXOpaqueType llvm::getPTXOpaqueType(const Argument &Arg) {
  if (argHasNVVMAnnotation(Arg, NVVMMetadata::Sampler))
    return PTXOpaqueType::Sampler;
  if (argHasNVVMAnnotation(Arg, NVVMMetadata::ReadOnlyImage))
    return PTXOpaqueType::Texture;
  if (argHasNVVMAnnotation(Arg, NVVMMetadata::WriteOnlyImage) ||
      argHasNVVMAnnotation(Arg, NVVMMetadata::ReadWriteImage))
    return PTXOpaqueType::Surface;
  return PTXOpaqueType::None;
}
 
PTXOpaqueType llvm::getPTXOpaqueType(const Value &V) {
  if (const auto *GV = dyn_cast<GlobalVariable>(&V))
    return getPTXOpaqueType(*GV);
  if (const auto *Arg = dyn_cast<Argument>(&V))
    return getPTXOpaqueType(*Arg);
  return PTXOpaqueType::None;
}
 
bool llvm::isManaged(const Value &V) {
  return globalHasNVVMAnnotation(V, NVVMMetadata::Managed);
}
 
SmallVector<unsigned, 3> llvm::getMaxNTID(const Function &F) {
  return getFnAttrParsedVector(F, NVVMAttr::MaxNTID);
}
 
SmallVector<unsigned, 3> llvm::getReqNTID(const Function &F) {
  return getFnAttrParsedVector(F, NVVMAttr::ReqNTID);
}
 
SmallVector<unsigned, 3> llvm::getClusterDim(const Function &F) {
  return getFnAttrParsedVector(F, NVVMAttr::ClusterDim);
}
 
std::optional<uint64_t> llvm::getOverallMaxNTID(const Function &F) {
  // Note: The semantics here are a bit strange. The PTX ISA states the
  // following (11.4.2. Performance-Tuning Directives: .maxntid):
  //
  //  Note that this directive guarantees that the total number of threads does
  //  not exceed the maximum, but does not guarantee that the limit in any
  //  particular dimension is not exceeded.
  return getVectorProduct(getMaxNTID(F));
}
 
std::optional<uint64_t> llvm::getOverallReqNTID(const Function &F) {
  // Note: The semantics here are a bit strange. See getOverallMaxNTID.
  return getVectorProduct(getReqNTID(F));
}
 
std::optional<uint64_t> llvm::getOverallClusterRank(const Function &F) {
  // maxclusterrank and cluster_dim are mutually exclusive.
  if (const auto ClusterRank = getMaxClusterRank(F))
    return ClusterRank;
 
  // Note: The semantics here are a bit strange. See getOverallMaxNTID.
  return getVectorProduct(getClusterDim(F));
}
 
std::optional<unsigned> llvm::getMaxClusterRank(const Function &F) {
  return getFnAttrParsedInt(F, NVVMAttr::MaxClusterRank);
}
 
std::optional<unsigned> llvm::getMinCTASm(const Function &F) {
  return getFnAttrParsedInt(F, NVVMAttr::MinCTASm);
}
 
std::optional<unsigned> llvm::getMaxNReg(const Function &F) {
  return getFnAttrParsedInt(F, NVVMAttr::MaxNReg);
}
 
bool llvm::hasBlocksAreClusters(const Function &F) {
  return F.hasFnAttribute(NVVMAttr::BlocksAreClusters);
}
 
bool llvm::isParamGridConstant(const Argument &Arg) {
  assert(isKernelFunction(*Arg.getParent()) &&
         "only kernel arguments can be grid_constant");
 
  if (!Arg.hasByValAttr())
    return false;
 
  // Lowering an argument as a grid_constant violates the byval semantics (and
  // the C++ API) by reusing the same memory location for the argument across
  // multiple threads. If an argument doesn't read memory and its address is not
  // captured (its address is not compared with any value), then the tweak of
  // the C++ API and byval semantics is unobservable by the program and we can
  // lower the arg as a grid_constant.
  if (Arg.onlyReadsMemory()) {
    const auto CI = Arg.getAttributes().getCaptureInfo();
    if (!capturesAddress(CI) && !capturesFullProvenance(CI))
      return true;
  }
 
  // "grid_constant" counts argument indices starting from 1
  return Arg.hasAttribute(NVVMAttr::GridConstant);
}
 
MaybeAlign llvm::getStackAlign(const CallBase &I, unsigned Index) {
  // First check the alignstack metadata.
  if (MaybeAlign StackAlign =
          I.getAttributes().getAttributes(Index).getStackAlignment())
    return StackAlign;
 
  // If that is missing, check the legacy nvvm metadata.
  if (MDNode *AlignNode = I.getMetadata("callalign")) {
    for (int I = 0, N = AlignNode->getNumOperands(); I < N; I++) {
      if (const auto *CI =
              mdconst::dyn_extract<ConstantInt>(AlignNode->getOperand(I))) {
        unsigned V = CI->getZExtValue();
        if ((V >> 16) == Index)
          return Align(V & 0xFFFF);
        if ((V >> 16) > Index)
          return std::nullopt;
      }
    }
  }
  return std::nullopt;
}