97 cl::desc(
"Lower GPU ctor / dtors to globals on the device."),
100#define DEPOTNAME "__local_depot"
110 if (
const User *U = dyn_cast<User>(V)) {
111 for (
unsigned i = 0, e = U->getNumOperands(); i != e; ++i) {
127 if (Visited.
count(GV))
131 if (!Visiting.
insert(GV).second)
149 NVPTX_MC::verifyInstructionPredicates(
MI->getOpcode(),
153 lowerToMCInst(
MI, Inst);
158bool NVPTXAsmPrinter::lowerImageHandleOperand(
const MachineInstr *
MI,
166 if (OpNo == 4 && MO.
isImm()) {
167 lowerImageHandleSymbol(MO.
getImm(), MCOp);
171 lowerImageHandleSymbol(MO.
getImm(), MCOp);
181 if (OpNo == VecSize && MO.
isImm()) {
182 lowerImageHandleSymbol(MO.
getImm(), MCOp);
189 if (OpNo == 0 && MO.
isImm()) {
190 lowerImageHandleSymbol(MO.
getImm(), MCOp);
197 if (OpNo == 1 && MO.
isImm()) {
198 lowerImageHandleSymbol(MO.
getImm(), MCOp);
208void NVPTXAsmPrinter::lowerImageHandleSymbol(
unsigned Index,
MCOperand &MCOp) {
221 if (
MI->getOpcode() == NVPTX::CALL_PROTOTYPE) {
229 for (
unsigned i = 0, e =
MI->getNumOperands(); i !=
e; ++i) {
234 if (lowerImageHandleOperand(
MI, i, MCOp)) {
240 if (lowerOperand(MO, MCOp))
294unsigned NVPTXAsmPrinter::encodeVirtualRegister(
unsigned Reg) {
299 unsigned RegNum = RegMap[
Reg];
304 if (RC == &NVPTX::Int1RegsRegClass) {
306 }
else if (RC == &NVPTX::Int16RegsRegClass) {
308 }
else if (RC == &NVPTX::Int32RegsRegClass) {
310 }
else if (RC == &NVPTX::Int64RegsRegClass) {
312 }
else if (RC == &NVPTX::Float32RegsRegClass) {
314 }
else if (RC == &NVPTX::Float64RegsRegClass) {
321 Ret |= (RegNum & 0x0FFFFFFF);
326 return Reg & 0x0FFFFFFF;
347 Type *Ty =
F->getReturnType();
359 if (
auto *ITy = dyn_cast<IntegerType>(Ty)) {
360 size = ITy->getBitWidth();
366 O <<
".param .b" <<
size <<
" func_retval0";
367 }
else if (isa<PointerType>(Ty)) {
368 O <<
".param .b" << TLI->getPointerTy(
DL).getSizeInBits()
371 unsigned totalsz =
DL.getTypeAllocSize(Ty);
372 unsigned retAlignment = 0;
374 retAlignment = TLI->getFunctionParamOptimizedAlign(F, Ty,
DL).value();
375 O <<
".param .align " << retAlignment <<
" .b8 func_retval0[" << totalsz
383 for (
unsigned i = 0, e = vtparts.
size(); i != e; ++i) {
385 EVT elemtype = vtparts[i];
387 elems = vtparts[i].getVectorNumElements();
388 elemtype = vtparts[i].getVectorElementType();
391 for (
unsigned j = 0, je = elems;
j != je; ++
j) {
395 O <<
".reg .b" << sz <<
" func_retval" << idx;
410 printReturnValStr(&F, O);
415bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll(
430 if (
const BasicBlock *PBB = PMBB->getBasicBlock()) {
432 PBB->getTerminator()->getMetadata(LLVMContext::MD_loop)) {
435 if (
MDNode *UnrollCountMD =
437 if (mdconst::extract<ConstantInt>(UnrollCountMD->getOperand(1))
449 if (isLoopHeaderOfNoUnroll(
MBB))
453void NVPTXAsmPrinter::emitFunctionEntryLabel() {
457 if (!GlobalsEmitted) {
459 GlobalsEmitted =
true;
465 emitLinkageDirective(F, O);
470 printReturnValStr(*
MF, O);
475 emitFunctionParamList(F, O);
479 emitKernelFunctionDirectives(*F, O);
489 setAndEmitFunctionVirtualRegisters(*
MF);
506void NVPTXAsmPrinter::emitFunctionBodyStart() {
513void NVPTXAsmPrinter::emitFunctionBodyEnd() {
523void NVPTXAsmPrinter::emitImplicitDef(
const MachineInstr *
MI)
const {
536void NVPTXAsmPrinter::emitKernelFunctionDirectives(
const Function &
F,
541 unsigned reqntidx, reqntidy, reqntidz;
542 bool specified =
false;
557 O <<
".reqntid " << reqntidx <<
", " << reqntidy <<
", " << reqntidz
563 unsigned maxntidx, maxntidy, maxntidz;
579 O <<
".maxntid " << maxntidx <<
", " << maxntidy <<
", " << maxntidz
584 O <<
".minnctapersm " << mincta <<
"\n";
588 O <<
".maxnreg " << maxnreg <<
"\n";
599 assert(
I != VRegMapping.
end() &&
"Bad register class");
603 assert(VI != RegMap.
end() &&
"Bad virtual register");
604 unsigned MappedVR = VI->second;
612void NVPTXAsmPrinter::emitVirtualRegister(
unsigned int vr,
618 emitLinkageDirective(F, O);
623 printReturnValStr(F, O);
626 emitFunctionParamList(F, O);
638 return GV->getName() !=
"llvm.used";
641 for (
const User *U :
C->users())
642 if (
const Constant *
C = dyn_cast<Constant>(U))
650 if (
const GlobalVariable *othergv = dyn_cast<GlobalVariable>(U)) {
651 if (othergv->getName() ==
"llvm.used")
656 if (
instr->getParent() &&
instr->getParent()->getParent()) {
658 if (oneFunc && (curFunc != oneFunc))
666 for (
const User *UU : U->users())
700 for (
const User *U :
C->users()) {
701 if (
const Constant *cu = dyn_cast<Constant>(U)) {
704 }
else if (
const Instruction *
I = dyn_cast<Instruction>(U)) {
721 if (
F.getAttributes().hasFnAttr(
"nvptx-libcall-callee")) {
722 emitDeclaration(&F, O);
726 if (
F.isDeclaration()) {
729 if (
F.getIntrinsicID())
731 emitDeclaration(&F, O);
734 for (
const User *U :
F.users()) {
735 if (
const Constant *
C = dyn_cast<Constant>(U)) {
740 emitDeclaration(&F, O);
746 emitDeclaration(&F, O);
751 if (!isa<Instruction>(U))
765 emitDeclaration(&F, O);
774 if (!GV)
return true;
776 if (!InitList)
return true;
780void NVPTXAsmPrinter::emitStartOfAsmFile(
Module &M) {
790 emitHeader(M, OS1, *STI);
804 "Module has a nontrivial global ctor, which NVPTX does not support.");
810 "Module has a nontrivial global dtor, which NVPTX does not support.");
817 GlobalsEmitted =
false;
822void NVPTXAsmPrinter::emitGlobals(
const Module &M) {
826 emitDeclarations(M, OS2);
841 assert(GVVisited.
size() == M.global_size() &&
"Missed a global variable");
842 assert(GVVisiting.
size() == 0 &&
"Did not fully process a global variable");
849 for (
unsigned i = 0, e = Globals.
size(); i != e; ++i)
850 printModuleLevelGV(Globals[i], OS2,
false, STI);
871 emitLinkageDirective(F,
OS);
873 printReturnValStr(F,
OS);
875 emitFunctionParamList(F,
OS);
880 OS <<
".alias " <<
Name->getName() <<
", " <<
F->getName() <<
";\n";
888 O <<
"// Generated by LLVM NVPTX Back-End\n";
893 O <<
".version " << (PTXVersion / 10) <<
"." << (PTXVersion % 10) <<
"\n";
900 O <<
", texmode_independent";
902 bool HasFullDebugInfo =
false;
904 switch(
CU->getEmissionKind()) {
910 HasFullDebugInfo =
true;
913 if (HasFullDebugInfo)
921 O <<
".address_size ";
936 if (!GlobalsEmitted) {
938 GlobalsEmitted =
true;
944 emitGlobalAlias(M, Alias);
949 A->eraseFromParent();
962 OutStreamer->emitRawText(
"\t.section\t.debug_loc\t{\t}");
966 TS->outputDwarfFileDirectives();
984void NVPTXAsmPrinter::emitLinkageDirective(
const GlobalValue *V,
987 if (V->hasExternalLinkage()) {
988 if (isa<GlobalVariable>(V)) {
996 }
else if (
V->isDeclaration())
1000 }
else if (
V->hasAppendingLinkage()) {
1002 msg.append(
"Error: ");
1003 msg.append(
"Symbol ");
1005 msg.append(std::string(
V->getName()));
1006 msg.append(
"has unsupported appending linkage type");
1008 }
else if (!
V->hasInternalLinkage() &&
1009 !
V->hasPrivateLinkage()) {
1015void NVPTXAsmPrinter::printModuleLevelGV(
const GlobalVariable *GVar,
1060 emitPTXGlobalVariable(GVar, O, STI);
1068 const Constant *Initializer =
nullptr;
1073 CI = dyn_cast<ConstantInt>(Initializer);
1082 O <<
"addr_mode_" << i <<
" = ";
1088 O <<
"clamp_to_border";
1091 O <<
"clamp_to_edge";
1102 O <<
"filter_mode = ";
1117 O <<
", force_unnormalized_coords = 1";
1127 if (strncmp(GVar->
getName().
data(),
"unrollpragma", 12) == 0)
1131 if (strncmp(GVar->
getName().
data(),
"filename", 8) == 0)
1137 const Function *demotedFunc =
nullptr;
1139 O <<
"// " << GVar->
getName() <<
" has been demoted\n";
1140 if (localDecls.find(demotedFunc) != localDecls.end())
1141 localDecls[demotedFunc].push_back(GVar);
1143 std::vector<const GlobalVariable *> temp;
1144 temp.push_back(GVar);
1145 localDecls[demotedFunc] = temp;
1151 emitPTXAddressSpace(PTy->getAddressSpace(), O);
1156 ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1158 O <<
" .attribute(.managed)";
1162 O <<
" .align " <<
A->value();
1164 O <<
" .align " << (int)
DL.getPrefTypeAlign(ETy).value();
1173 O << getPTXFundamentalTypeStr(ETy,
false);
1184 if (!Initializer->
isNullValue() && !isa<UndefValue>(Initializer)) {
1186 printScalarConstant(Initializer, O);
1195 "' is not allowed in addrspace(" +
1196 Twine(PTy->getAddressSpace()) +
")");
1212 ElementSize =
DL.getTypeStoreSize(ETy);
1219 if (!isa<UndefValue>(Initializer) && !Initializer->
isNullValue()) {
1220 AggBuffer aggBuffer(ElementSize, *
this);
1221 bufferAggregateConstant(Initializer, &aggBuffer);
1222 if (aggBuffer.numSymbols()) {
1224 if (ElementSize % ptrSize ||
1225 !aggBuffer.allSymbolsAligned(ptrSize)) {
1229 "initialized packed aggregate with pointers '" +
1231 "' requires at least PTX ISA version 7.1");
1234 O <<
"[" << ElementSize <<
"] = {";
1235 aggBuffer.printBytes(O);
1238 O <<
" .u" << ptrSize * 8 <<
" ";
1240 O <<
"[" << ElementSize / ptrSize <<
"] = {";
1241 aggBuffer.printWords(O);
1247 O <<
"[" << ElementSize <<
"] = {";
1248 aggBuffer.printBytes(O);
1277void NVPTXAsmPrinter::AggBuffer::printSymbol(
unsigned nSym,
raw_ostream &os) {
1278 const Value *
v = Symbols[nSym];
1279 const Value *v0 = SymbolsBeforeStripping[nSym];
1280 if (
const GlobalValue *GVar = dyn_cast<GlobalValue>(v)) {
1284 bool isGenericPointer = PTy && PTy->getAddressSpace() == 0;
1285 if (EmitGeneric && isGenericPointer && !isa<Function>(v)) {
1287 Name->print(os, AP.MAI);
1290 Name->print(os, AP.MAI);
1292 }
else if (
const ConstantExpr *CExpr = dyn_cast<ConstantExpr>(v0)) {
1293 const MCExpr *Expr = AP.lowerConstantForGV(cast<Constant>(CExpr),
false);
1294 AP.printMCExpr(*Expr, os);
1299void NVPTXAsmPrinter::AggBuffer::printBytes(
raw_ostream &os) {
1300 unsigned int ptrSize = AP.MAI->getCodePointerSize();
1301 symbolPosInBuffer.push_back(size);
1302 unsigned int nSym = 0;
1303 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1304 for (
unsigned int pos = 0; pos <
size;) {
1307 if (pos != nextSymbolPos) {
1308 os << (
unsigned int)buffer[pos];
1315 std::string symText;
1317 printSymbol(nSym, oss);
1318 for (
unsigned i = 0; i < ptrSize; ++i) {
1322 os <<
"(" << symText <<
")";
1325 nextSymbolPos = symbolPosInBuffer[++nSym];
1326 assert(nextSymbolPos >= pos);
1330void NVPTXAsmPrinter::AggBuffer::printWords(
raw_ostream &os) {
1331 unsigned int ptrSize = AP.MAI->getCodePointerSize();
1332 symbolPosInBuffer.push_back(size);
1333 unsigned int nSym = 0;
1334 unsigned int nextSymbolPos = symbolPosInBuffer[nSym];
1335 assert(nextSymbolPos % ptrSize == 0);
1336 for (
unsigned int pos = 0; pos <
size; pos += ptrSize) {
1339 if (pos == nextSymbolPos) {
1340 printSymbol(nSym, os);
1341 nextSymbolPos = symbolPosInBuffer[++nSym];
1342 assert(nextSymbolPos % ptrSize == 0);
1343 assert(nextSymbolPos >= pos + ptrSize);
1344 }
else if (ptrSize == 4)
1352 if (localDecls.find(f) == localDecls.end())
1355 std::vector<const GlobalVariable *> &gvars = localDecls[
f];
1362 O <<
"\t// demoted variable\n\t";
1363 printModuleLevelGV(GV, O,
true, STI);
1367void NVPTXAsmPrinter::emitPTXAddressSpace(
unsigned int AddressSpace,
1390NVPTXAsmPrinter::getPTXFundamentalTypeStr(
Type *Ty,
bool useB4PTR)
const {
1393 unsigned NumBits = cast<IntegerType>(Ty)->getBitWidth();
1396 else if (NumBits <= 64) {
1397 std::string
name =
"u";
1398 return name + utostr(NumBits);
1416 assert((PtrSize == 64 || PtrSize == 32) &&
"Unexpected pointer size");
1434void NVPTXAsmPrinter::emitPTXGlobalVariable(
const GlobalVariable *GVar,
1447 ".attribute(.managed) requires PTX version >= 4.0 and sm_30");
1449 O <<
" .attribute(.managed)";
1452 O <<
" .align " <<
A->value();
1454 O <<
" .align " << (int)
DL.getPrefTypeAlign(ETy).value();
1466 O << getPTXFundamentalTypeStr(ETy);
1472 int64_t ElementSize = 0;
1482 ElementSize =
DL.getTypeStoreSize(ETy);
1503 unsigned paramIndex = 0;
1509 if (
F->arg_empty() && !
F->isVarArg()) {
1516 for (
I =
F->arg_begin(),
E =
F->arg_end();
I !=
E; ++
I, paramIndex++) {
1517 Type *Ty =
I->getType();
1528 std::string sname = std::string(
I->getName());
1530 if (hasImageHandles)
1531 O <<
"\t.param .u64 .ptr .surfref ";
1533 O <<
"\t.param .surfref ";
1534 O << TLI->getParamName(F, paramIndex);
1537 if (hasImageHandles)
1538 O <<
"\t.param .u64 .ptr .texref ";
1540 O <<
"\t.param .texref ";
1541 O << TLI->getParamName(F, paramIndex);
1544 if (hasImageHandles)
1545 O <<
"\t.param .u64 .ptr .samplerref ";
1547 O <<
"\t.param .samplerref ";
1548 O << TLI->getParamName(F, paramIndex);
1554 auto getOptimalAlignForParam = [TLI, &
DL, &PAL,
F,
1556 Align TypeAlign = TLI->getFunctionParamOptimizedAlign(F, Ty,
DL);
1557 MaybeAlign ParamAlign = PAL.getParamAlignment(paramIndex);
1558 return std::max(TypeAlign, ParamAlign.
valueOrOne());
1561 if (!PAL.hasParamAttr(paramIndex, Attribute::ByVal)) {
1567 Align OptimalAlign = getOptimalAlignForParam(Ty);
1569 O <<
"\t.param .align " << OptimalAlign.
value() <<
" .b8 ";
1570 O << TLI->getParamName(F, paramIndex);
1571 O <<
"[" <<
DL.getTypeAllocSize(Ty) <<
"]";
1576 auto *PTy = dyn_cast<PointerType>(Ty);
1577 unsigned PTySizeInBits = 0;
1580 TLI->getPointerTy(
DL, PTy->getAddressSpace()).getSizeInBits();
1581 assert(PTySizeInBits &&
"Invalid pointer size");
1587 O <<
"\t.param .u" << PTySizeInBits <<
" ";
1591 int addrSpace = PTy->getAddressSpace();
1592 switch (addrSpace) {
1597 O <<
".ptr .const ";
1600 O <<
".ptr .shared ";
1603 O <<
".ptr .global ";
1606 Align ParamAlign =
I->getParamAlign().valueOrOne();
1607 O <<
".align " << ParamAlign.
value() <<
" ";
1609 O << TLI->getParamName(F, paramIndex);
1619 O << getPTXFundamentalTypeStr(Ty);
1621 O << TLI->getParamName(F, paramIndex);
1627 if (isa<IntegerType>(Ty)) {
1628 sz = cast<IntegerType>(Ty)->getBitWidth();
1631 assert(PTySizeInBits &&
"Invalid pointer size");
1636 O <<
"\t.param .b" << sz <<
" ";
1638 O <<
"\t.reg .b" << sz <<
" ";
1639 O << TLI->getParamName(F, paramIndex);
1644 Type *ETy = PAL.getParamByValType(paramIndex);
1645 assert(ETy &&
"Param should have byval type");
1647 if (isABI || isKernelFunc) {
1652 Align OptimalAlign =
1654 ? getOptimalAlignForParam(ETy)
1655 : TLI->getFunctionByValParamAlign(
1656 F, ETy, PAL.getParamAlignment(paramIndex).valueOrOne(),
DL);
1658 unsigned sz =
DL.getTypeAllocSize(ETy);
1659 O <<
"\t.param .align " << OptimalAlign.
value() <<
" .b8 ";
1660 O << TLI->getParamName(F, paramIndex);
1661 O <<
"[" << sz <<
"]";
1670 for (
unsigned i = 0, e = vtparts.
size(); i != e; ++i) {
1672 EVT elemtype = vtparts[i];
1674 elems = vtparts[i].getVectorNumElements();
1675 elemtype = vtparts[i].getVectorElementType();
1678 for (
unsigned j = 0, je = elems;
j != je; ++
j) {
1682 O <<
"\t.reg .b" << sz <<
" ";
1683 O << TLI->getParamName(F, paramIndex);
1696 if (
F->isVarArg()) {
1701 O << TLI->getParamName(F, -1) <<
"[]";
1707void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
1724 O <<
"\t.reg .b64 \t%SP;\n";
1725 O <<
"\t.reg .b64 \t%SPL;\n";
1727 O <<
"\t.reg .b32 \t%SP;\n";
1728 O <<
"\t.reg .b32 \t%SPL;\n";
1737 for (
unsigned i = 0; i < numVRs; i++) {
1741 int n = regmap.
size();
1742 regmap.
insert(std::make_pair(vr, n + 1));
1757 for (
unsigned i=0; i<
TRI->getNumRegClasses(); i++) {
1762 int n = regmap.
size();
1766 O <<
"\t.reg " << rcname <<
" \t" << rcStr <<
"<" << (n+1)
1777 unsigned int numHex;
1796 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(CPV)) {
1800 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CPV)) {
1801 printFPConstant(CFP, O);
1804 if (isa<ConstantPointerNull>(CPV)) {
1808 if (
const GlobalValue *GVar = dyn_cast<GlobalValue>(CPV)) {
1809 bool IsNonGenericPointer =
false;
1811 IsNonGenericPointer =
true;
1813 if (EmitGeneric && !isa<Function>(CPV) && !IsNonGenericPointer) {
1822 if (
const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(CPV)) {
1823 const MCExpr *
E = lowerConstantForGV(cast<Constant>(Cexpr),
false);
1830void NVPTXAsmPrinter::bufferLEByte(
const Constant *CPV,
int Bytes,
1831 AggBuffer *AggBuffer) {
1833 int AllocSize =
DL.getTypeAllocSize(CPV->
getType());
1837 AggBuffer->addZeros(Bytes ? Bytes : AllocSize);
1842 auto AddIntToBuffer = [AggBuffer, Bytes](
const APInt &Val) {
1843 size_t NumBytes = (Val.getBitWidth() + 7) / 8;
1845 for (
unsigned I = 0;
I < NumBytes; ++
I) {
1846 Buf[
I] = Val.extractBitsAsZExtValue(8,
I * 8);
1848 AggBuffer->addBytes(Buf.data(), NumBytes, Bytes);
1853 if (
const auto CI = dyn_cast<ConstantInt>(CPV)) {
1857 if (
const auto *Cexpr = dyn_cast<ConstantExpr>(CPV)) {
1858 if (
const auto *CI =
1863 if (Cexpr->getOpcode() == Instruction::PtrToInt) {
1864 Value *
V = Cexpr->getOperand(0)->stripPointerCasts();
1865 AggBuffer->addSymbol(V, Cexpr->getOperand(0));
1866 AggBuffer->addZeros(AllocSize);
1877 AddIntToBuffer(cast<ConstantFP>(CPV)->getValueAPF().bitcastToAPInt());
1881 if (
const GlobalValue *GVar = dyn_cast<GlobalValue>(CPV)) {
1882 AggBuffer->addSymbol(GVar, GVar);
1883 }
else if (
const ConstantExpr *Cexpr = dyn_cast<ConstantExpr>(CPV)) {
1884 const Value *
v = Cexpr->stripPointerCasts();
1885 AggBuffer->addSymbol(v, Cexpr);
1887 AggBuffer->addZeros(AllocSize);
1894 if (isa<ConstantAggregate>(CPV) || isa<ConstantDataSequential>(CPV)) {
1895 bufferAggregateConstant(CPV, AggBuffer);
1896 if (Bytes > AllocSize)
1897 AggBuffer->addZeros(Bytes - AllocSize);
1898 }
else if (isa<ConstantAggregateZero>(CPV))
1899 AggBuffer->addZeros(Bytes);
1910void NVPTXAsmPrinter::bufferAggregateConstant(
const Constant *CPV,
1911 AggBuffer *aggBuffer) {
1916 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(CPV)) {
1918 for (
unsigned I = 0,
E =
DL.getTypeAllocSize(CPV->
getType());
I <
E; ++
I) {
1920 aggBuffer->addBytes(&Byte, 1, 1);
1927 if (isa<ConstantArray>(CPV) || isa<ConstantVector>(CPV)) {
1930 bufferLEByte(cast<Constant>(CPV->
getOperand(i)), 0, aggBuffer);
1935 dyn_cast<ConstantDataSequential>(CPV)) {
1936 if (CDS->getNumElements())
1937 for (
unsigned i = 0; i < CDS->getNumElements(); ++i)
1938 bufferLEByte(cast<Constant>(CDS->getElementAsConstant(i)), 0,
1943 if (isa<ConstantStruct>(CPV)) {
1948 Bytes =
DL.getStructLayout(ST)->getElementOffset(0) +
1949 DL.getTypeAllocSize(ST) -
1950 DL.getStructLayout(ST)->getElementOffset(i);
1952 Bytes =
DL.getStructLayout(ST)->getElementOffset(i + 1) -
1953 DL.getStructLayout(ST)->getElementOffset(i);
1954 bufferLEByte(cast<Constant>(CPV->
getOperand(i)), Bytes, aggBuffer);
1967NVPTXAsmPrinter::lowerConstantForGV(
const Constant *CV,
bool ProcessingGeneric) {
1973 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(CV))
1976 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
1979 if (ProcessingGeneric) {
1991 switch (
CE->getOpcode()) {
1998 return lowerConstantForGV(
C, ProcessingGeneric);
2003 OS <<
"Unsupported expression in static initializer: ";
2004 CE->printAsOperand(
OS,
false,
2009 case Instruction::AddrSpaceCast: {
2012 if (DstTy->getAddressSpace() == 0) {
2013 return lowerConstantForGV(cast<const Constant>(
CE->getOperand(0)),
true);
2017 OS <<
"Unsupported expression in static initializer: ";
2018 CE->printAsOperand(
OS,
false,
2023 case Instruction::GetElementPtr: {
2027 APInt OffsetAI(
DL.getPointerTypeSizeInBits(
CE->getType()), 0);
2028 cast<GEPOperator>(CE)->accumulateConstantOffset(
DL, OffsetAI);
2030 const MCExpr *
Base = lowerConstantForGV(
CE->getOperand(0),
2035 int64_t
Offset = OffsetAI.getSExtValue();
2040 case Instruction::Trunc:
2046 case Instruction::BitCast:
2047 return lowerConstantForGV(
CE->getOperand(0), ProcessingGeneric);
2049 case Instruction::IntToPtr: {
2057 return lowerConstantForGV(
Op, ProcessingGeneric);
2060 case Instruction::PtrToInt: {
2066 Type *Ty =
CE->getType();
2068 const MCExpr *OpExpr = lowerConstantForGV(
Op, ProcessingGeneric);
2072 if (
DL.getTypeAllocSize(Ty) ==
DL.getTypeAllocSize(
Op->getType()))
2078 unsigned InBits =
DL.getTypeAllocSizeInBits(
Op->getType());
2085 case Instruction::Add: {
2086 const MCExpr *
LHS = lowerConstantForGV(
CE->getOperand(0), ProcessingGeneric);
2087 const MCExpr *
RHS = lowerConstantForGV(
CE->getOperand(1), ProcessingGeneric);
2088 switch (
CE->getOpcode()) {
2100 return cast<MCTargetExpr>(&Expr)->printImpl(
OS,
MAI);
2102 OS << cast<MCConstantExpr>(Expr).getValue();
2128 if (isa<MCConstantExpr>(BE.
getLHS()) || isa<MCSymbolRefExpr>(BE.
getLHS()) ||
2129 isa<NVPTXGenericMCSymbolRefExpr>(BE.
getLHS())) {
2141 if (RHSC->getValue() < 0) {
2142 OS << RHSC->getValue();
2153 if (isa<MCConstantExpr>(BE.
getRHS()) || isa<MCSymbolRefExpr>(BE.
getRHS())) {
2169bool NVPTXAsmPrinter::PrintAsmOperand(
const MachineInstr *
MI,
unsigned OpNo,
2171 if (ExtraCode && ExtraCode[0]) {
2172 if (ExtraCode[1] != 0)
2175 switch (ExtraCode[0]) {
2184 printOperand(
MI, OpNo, O);
2189bool NVPTXAsmPrinter::PrintAsmMemoryOperand(
const MachineInstr *
MI,
2191 const char *ExtraCode,
2193 if (ExtraCode && ExtraCode[0])
2197 printMemOperand(
MI, OpNo, O);
2203void NVPTXAsmPrinter::printOperand(
const MachineInstr *
MI,
unsigned OpNum,
2209 if (MO.
getReg() == NVPTX::VRDepot)
2214 emitVirtualRegister(MO.
getReg(), O);
2239void NVPTXAsmPrinter::printMemOperand(
const MachineInstr *
MI,
unsigned OpNum,
2241 printOperand(
MI, OpNum, O);
2243 if (Modifier && strcmp(Modifier,
"add") == 0) {
2245 printOperand(
MI, OpNum + 1, O);
2247 if (
MI->getOperand(OpNum + 1).isImm() &&
2248 MI->getOperand(OpNum + 1).getImm() == 0)
2251 printOperand(
MI, OpNum + 1, O);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< bool > LowerCtorDtor("amdgpu-lower-global-ctor-dtor", cl::desc("Lower GPU ctor / dtors to globals on the device."), cl::init(true), cl::Hidden)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static bool isEmptyXXStructor(GlobalVariable *GV)
static bool usedInOneFunc(const User *U, Function const *&oneFunc)
static void VisitGlobalVariableForEmission(const GlobalVariable *GV, SmallVectorImpl< const GlobalVariable * > &Order, DenseSet< const GlobalVariable * > &Visited, DenseSet< const GlobalVariable * > &Visiting)
VisitGlobalVariableForEmission - Add GV to the list of GlobalVariable instances to be emitted,...
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXAsmPrinter()
static bool usedInGlobalVarDef(const Constant *C)
static bool useFuncSeen(const Constant *C, DenseMap< const Function *, bool > &seenMap)
static cl::opt< bool > LowerCtorDtor("nvptx-lower-global-ctor-dtor", cl::desc("Lower GPU ctor / dtors to globals on the device."), cl::init(false), cl::Hidden)
static bool ShouldPassAsArray(Type *Ty)
static void DiscoverDependentGlobals(const Value *V, DenseSet< const GlobalVariable * > &Globals)
DiscoverDependentGlobals - Return a set of GlobalVariables on which V depends.
static bool canDemoteGlobalVar(const GlobalVariable *gv, Function const *&f)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallString class.
This file defines the SmallVector class.
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
uint64_t getZExtValue() const
Get zero extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
This class represents an incoming formal argument to a Function.
MCSymbol * getSymbol(const GlobalValue *GV) const
void EmitToStreamer(MCStreamer &S, const MCInst &Inst)
TargetMachine & TM
Target machine description.
virtual void PrintSymbolOperand(const MachineOperand &MO, raw_ostream &OS)
Print the MachineOperand as a symbol.
const MCAsmInfo * MAI
Target Asm Printer information.
MachineFunction * MF
The current machine function.
bool doInitialization(Module &M) override
Set up the AsmPrinter when we are working on a new module.
unsigned getFunctionNumber() const
Return a unique ID for the current function.
MCSymbol * CurrentFnSym
The symbol for the current function.
MachineModuleInfo * MMI
This is a pointer to the current MachineModuleInfo.
MCContext & OutContext
This is the context for the output file that we are streaming.
bool doFinalization(Module &M) override
Shut down the asmprinter.
MCSymbol * GetExternalSymbolSymbol(StringRef Sym) const
Return the MCSymbol for the specified ExternalSymbol.
virtual void emitBasicBlockStart(const MachineBasicBlock &MBB)
Targets can override this to emit stuff at the start of a basic block.
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
std::unique_ptr< MCStreamer > OutStreamer
This is the MCStreamer object for the file we are generating.
const DataLayout & getDataLayout() const
Return information about data layout.
void emitInitialRawDwarfLocDirective(const MachineFunction &MF)
Emits inital debug location directive.
const MCSubtargetInfo & getSubtargetInfo() const
Return information about subtarget.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
ConstantArray - Constant Array Declarations.
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
A constant value that is initialized with an expression using other constant values.
static Constant * getIntegerCast(Constant *C, Type *Ty, bool IsSigned)
Create a ZExt, Bitcast or Trunc for integer -> integer casts.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important base class in LLVM.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
const Constant * getAliasee() const
StringRef getSection() const
Get the custom section of this global if it has one.
MaybeAlign getAlign() const
Returns the alignment of the given variable or function.
bool hasSection() const
Check if this global has a custom object file section.
bool hasLinkOnceLinkage() const
bool hasExternalLinkage() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
bool hasWeakLinkage() const
bool hasCommonLinkage() const
bool hasAvailableExternallyLinkage() const
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
This class describes a target machine that is implemented with the LLVM target-independent code gener...
unsigned getCodePointerSize() const
Get the code pointer size in bytes.
Binary assembler expressions.
const MCExpr * getLHS() const
Get the left-hand side expression of the binary operator.
const MCExpr * getRHS() const
Get the right-hand side expression of the binary operator.
static const MCBinaryExpr * createAnd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Opcode getOpcode() const
Get the kind of this binary expression.
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Base class for the full range of assembler expressions which are needed for parsing.
@ Unary
Unary expressions.
@ Constant
Constant expressions.
@ SymbolRef
References to labels and assigned expressions.
@ Target
Target specific expression.
@ Binary
Binary expressions.
Instances of this class represent a single low-level machine instruction.
void addOperand(const MCOperand Op)
void setOpcode(unsigned Op)
Describe properties that are true of each instruction in the target description file.
Instances of this class represent operands of the MCInst class.
static MCOperand createReg(unsigned Reg)
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createImm(int64_t Val)
Represent a reference to a symbol from inside an expression.
const MCSymbol & getSymbol() const
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
void print(raw_ostream &OS, const MCAsmInfo *MAI) const
print - Print the value to the stream OS.
Unary assembler expressions.
Opcode getOpcode() const
Get the kind of this unary expression.
const MCExpr * getSubExpr() const
Get the child of this unary expression.
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
iterator_range< pred_iterator > predecessors()
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
bool isLoopHeader(const MachineBasicBlock *BB) const
True if the block is a loop header node.
MachineLoop * getLoopFor(const MachineBasicBlock *BB) const
Return the innermost loop that BB lives in.
bool hasDebugInfo() const
Returns true if valid debug info is present.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
const char * getSymbolName() const
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_MachineBasicBlock
MachineBasicBlock reference.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
@ MO_FPImmediate
Floating-point immediate operand.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
A Module instance is used to store all the information related to an LLVM module.
bool doInitialization(Module &M) override
Set up the AsmPrinter when we are working on a new module.
bool runOnMachineFunction(MachineFunction &F) override
Emit the specified function out to the OutStreamer.
std::string getVirtualRegisterName(unsigned) const
bool doFinalization(Module &M) override
Shut down the asmprinter.
const MCSymbol * getFunctionFrameSymbol() const override
Return symbol for the function pseudo stack if the stack frame is not a register based.
static const NVPTXFloatMCExpr * createConstantBFPHalf(const APFloat &Flt, MCContext &Ctx)
static const NVPTXFloatMCExpr * createConstantFPHalf(const APFloat &Flt, MCContext &Ctx)
static const NVPTXFloatMCExpr * createConstantFPSingle(const APFloat &Flt, MCContext &Ctx)
static const NVPTXFloatMCExpr * createConstantFPDouble(const APFloat &Flt, MCContext &Ctx)
static const NVPTXGenericMCSymbolRefExpr * create(const MCSymbolRefExpr *SymExpr, MCContext &Ctx)
static const char * getRegisterName(MCRegister Reg)
const char * getImageHandleSymbol(unsigned Idx) const
Returns the symbol name at the given index.
const char * getName(unsigned RegNo) const
std::string getTargetName() const
bool hasImageHandles() const
unsigned getMaxRequiredAlignment() const
bool hasMaskOperator() const
const NVPTXTargetLowering * getTargetLowering() const override
unsigned getPTXVersion() const
const NVPTXRegisterInfo * getRegisterInfo() const override
unsigned int getSmVersion() const
NVPTX::DrvInterface getDrvInterface() const
const NVPTXSubtarget * getSubtargetImpl(const Function &) const override
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
UniqueStringSaver & getStrPool() const
Implments NVPTX-specific streamer.
void closeLastSection()
Close last section.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool startswith(StringRef Prefix) const
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Class to represent struct types.
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
unsigned getPointerSizeInBits(unsigned AS) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
@ HalfTyID
16-bit floating point type
@ VoidTyID
type with no size
@ FloatTyID
32-bit floating point type
@ IntegerTyID
Arbitrary bit width integers.
@ FixedVectorTyID
Fixed width SIMD vector type.
@ BFloatTyID
16-bit floating point type (7-bit significand)
@ DoubleTyID
64-bit floating point type
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isAggregateType() const
Return true if the type is an aggregate type.
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
StringRef save(const char *S)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
bool erase(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
@ CE
Windows NT (Windows on ARM)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
uint64_t read64le(const void *P)
uint32_t read32le(const void *P)
This is an optimization pass for GlobalISel generic memory operations.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< TypeSize > *Offsets, TypeSize StartingOffset)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
std::string getSamplerName(const Value &val)
bool getAlign(const Function &F, unsigned index, unsigned &align)
bool getMinCTASm(const Function &F, unsigned &x)
std::string getNVPTXRegClassName(TargetRegisterClass const *RC)
bool isImage(const Value &val)
bool getMaxNTIDz(const Function &F, unsigned &z)
Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
bool isManaged(const Value &val)
unsigned promoteScalarArgumentSize(unsigned size)
bool isSurface(const Value &val)
void clearAnnotationCache(const Module *Mod)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
std::string getSurfaceName(const Value &val)
bool getReqNTIDx(const Function &F, unsigned &x)
bool getReqNTIDy(const Function &F, unsigned &y)
FormattedNumber format_hex_no_prefix(uint64_t N, unsigned Width, bool Upper=false)
format_hex_no_prefix - Output N as a fixed width hexadecimal.
bool getMaxNReg(const Function &F, unsigned &x)
bool isTexture(const Value &val)
bool isImageWriteOnly(const Value &val)
bool isImageReadWrite(const Value &val)
void write_hex(raw_ostream &S, uint64_t N, HexPrintStyle Style, std::optional< size_t > Width=std::nullopt)
std::string getTextureName(const Value &val)
std::string getNVPTXRegClassStr(TargetRegisterClass const *RC)
Target & getTheNVPTXTarget64()
bool isKernelFunction(const Function &F)
bool getReqNTIDz(const Function &F, unsigned &z)
bool getMaxNTIDx(const Function &F, unsigned &x)
bool getMaxNTIDy(const Function &F, unsigned &y)
bool isSampler(const Value &val)
MDNode * GetUnrollMetadata(MDNode *LoopID, StringRef Name)
Given an llvm.loop loop id metadata node, returns the loop hint metadata node with the given name (fo...
Target & getTheNVPTXTarget32()
static const fltSemantics & IEEEsingle() LLVM_READNONE
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...