24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/ADT/StringMap.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/OnDiskHashTable.h"
30 #include "llvm/Support/Path.h"
31 #include "llvm/Support/raw_ostream.h"
35 #define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
38 using namespace clang;
48 Offset TokenData, PPCondData;
54 : TokenData(td), PPCondData(ppcd) {}
56 Offset getTokenOffset()
const {
return TokenData; }
57 Offset getPPCondTableOffset()
const {
return PPCondData; }
61 class PTHEntryKeyVariant {
62 union {
const FileEntry* FE;
const char* Path; };
63 enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 }
Kind;
67 PTHEntryKeyVariant(
const FileEntry *fe) : FE(fe),
Kind(IsFE), Data(
nullptr) {}
69 PTHEntryKeyVariant(
FileData *Data,
const char *path)
72 explicit PTHEntryKeyVariant(
const char *path)
73 : Path(path),
Kind(IsNoExist), Data(
nullptr) {}
75 bool isFile()
const {
return Kind == IsFE; }
77 StringRef getString()
const {
78 return Kind == IsFE ? FE->getName() : Path;
81 unsigned getKind()
const {
return (
unsigned)
Kind; }
83 void EmitData(raw_ostream& Out) {
84 using namespace llvm::support;
85 endian::Writer<little> LE(Out);
89 llvm::sys::fs::UniqueID UID = FE->getUniqueID();
90 LE.write<uint64_t>(UID.getFile());
91 LE.write<uint64_t>(UID.getDevice());
92 LE.write<uint64_t>(FE->getModificationTime());
93 LE.write<uint64_t>(FE->getSize());
97 LE.write<uint64_t>(Data->
UniqueID.getFile());
98 LE.write<uint64_t>(Data->
UniqueID.getDevice());
99 LE.write<uint64_t>(Data->
ModTime);
100 LE.write<uint64_t>(Data->
Size);
108 unsigned getRepresentationLength()
const {
109 return Kind == IsNoExist ? 0 : 4 * 8;
113 class FileEntryPTHEntryInfo {
115 typedef PTHEntryKeyVariant key_type;
116 typedef key_type key_type_ref;
118 typedef PTHEntry data_type;
119 typedef const PTHEntry& data_type_ref;
121 typedef unsigned hash_value_type;
122 typedef unsigned offset_type;
124 static hash_value_type
ComputeHash(PTHEntryKeyVariant V) {
125 return llvm::HashString(V.getString());
128 static std::pair<unsigned,unsigned>
129 EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
131 using namespace llvm::support;
132 endian::Writer<little> LE(Out);
134 unsigned n = V.getString().size() + 1 + 1;
135 LE.write<uint16_t>(n);
137 unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
138 LE.write<uint8_t>(m);
140 return std::make_pair(n, m);
143 static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V,
unsigned n){
144 using namespace llvm::support;
146 endian::Writer<little>(Out).write<uint8_t>((
unsigned)V.getKind());
148 Out.write(V.getString().data(), n - 1);
151 static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
152 const PTHEntry& E,
unsigned) {
153 using namespace llvm::support;
154 endian::Writer<little> LE(Out);
159 LE.write<uint32_t>(E.getTokenOffset());
160 LE.write<uint32_t>(E.getPPCondTableOffset());
172 OffsetOpt() : valid(
false) {}
173 bool hasOffset()
const {
return valid; }
174 Offset getOffset()
const { assert(valid);
return off; }
175 void setOffset(
Offset o) { off = o; valid =
true; }
179 typedef llvm::OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo>
PTHMap;
183 typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
184 typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
187 raw_pwrite_stream &Out;
191 CachedStrsTy CachedStrs;
193 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
199 void EmitToken(
const Token& T);
201 void Emit8(uint32_t V) {
202 using namespace llvm::support;
203 endian::Writer<little>(Out).write<uint8_t>(V);
206 void Emit16(uint32_t V) {
207 using namespace llvm::support;
208 endian::Writer<little>(Out).write<uint16_t>(V);
211 void Emit32(uint32_t V) {
212 using namespace llvm::support;
213 endian::Writer<little>(Out).write<uint32_t>(V);
216 void EmitBuf(
const char *Ptr,
unsigned NumBytes) {
217 Out.write(Ptr, NumBytes);
221 using namespace llvm::support;
222 endian::Writer<little>(Out).write<uint16_t>(V.size());
223 EmitBuf(V.data(), V.size());
230 std::pair<Offset, Offset> EmitIdentifierTable();
234 Offset EmitFileTable() {
return PM.Emit(Out); }
236 PTHEntry LexTokens(
Lexer& L);
237 Offset EmitCachedSpellings();
241 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
243 PTHMap &getPM() {
return PM; }
261 void PTHWriter::EmitToken(
const Token& T) {
264 (((uint32_t) T.getLength()) << 16));
274 auto &
E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first;
277 if (!E.second.hasOffset()) {
278 E.second.setOffset(CurStrOffset);
279 StrEntries.push_back(&E);
280 CurStrOffset += s.size() + 1;
284 Emit32(E.second.getOffset());
289 Emit32(PP.getSourceManager().getFileOffset(T.
getLocation()));
292 PTHEntry PTHWriter::LexTokens(
Lexer& L) {
295 using namespace llvm::support;
296 endian::Writer<little> LE(Out);
297 uint32_t TokenOff = Out.tell();
298 for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
299 LE.write<uint8_t>(0);
302 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
304 std::vector<unsigned> PPStartCond;
305 bool ParsingPreprocessorDirective =
false;
312 if ((Tok.isAtStartOfLine() || Tok.is(
tok::eof)) &&
313 ParsingPreprocessorDirective) {
323 ParsingPreprocessorDirective =
false;
326 if (Tok.is(tok::raw_identifier)) {
327 PP.LookUpIdentifierInfo(Tok);
332 if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
335 assert(!ParsingPreprocessorDirective);
352 if (Tok.isNot(tok::raw_identifier)) {
360 ParsingPreprocessorDirective =
true;
363 case tok::pp_not_keyword:
369 case tok::pp_include:
371 case tok::pp_include_next: {
378 assert(!Tok.isAtStartOfLine());
379 if (Tok.is(tok::raw_identifier))
380 PP.LookUpIdentifierInfo(Tok);
386 case tok::pp_ifndef: {
389 PPStartCond.push_back(PPCond.size());
390 PPCond.push_back(std::make_pair(HashOff, 0U));
393 case tok::pp_endif: {
397 unsigned index = PPCond.size();
399 assert(!PPStartCond.empty());
400 assert(PPCond.size() > PPStartCond.back());
401 assert(PPCond[PPStartCond.back()].second == 0);
402 PPCond[PPStartCond.back()].second = index;
403 PPStartCond.pop_back();
405 PPCond.push_back(std::make_pair(HashOff, index));
412 while (Tok.isNot(
tok::eof) && !Tok.isAtStartOfLine());
422 unsigned index = PPCond.size();
424 assert(!PPStartCond.empty());
425 assert(PPCond.size() > PPStartCond.back());
426 assert(PPCond[PPStartCond.back()].second == 0);
427 PPCond[PPStartCond.back()].second = index;
428 PPStartCond.pop_back();
430 PPCond.push_back(std::make_pair(HashOff, 0U));
431 PPStartCond.push_back(index);
441 assert(PPStartCond.empty() &&
"Error: imblanced preprocessor conditionals.");
447 Emit32(PPCond.size());
449 for (
unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
450 Emit32(PPCond[i].first - TokenOff);
451 uint32_t x = PPCond[i].second;
452 assert(x != 0 &&
"PPCond entry not backpatched.");
455 Emit32(x == i ? 0 : x);
458 return PTHEntry(TokenOff, PPCondOff);
461 Offset PTHWriter::EmitCachedSpellings() {
463 Offset SpellingsOff = Out.tell();
465 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::
iterator
466 I = StrEntries.begin(), E = StrEntries.end(); I!=
E; ++
I)
467 EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 );
473 return llvm::support::endian::byte_swap<uint32_t, llvm::support::little>(
X);
476 static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) {
478 OS.pwrite(reinterpret_cast<const char *>(&LEVal), 4, Off);
484 Out <<
"cfe-pth" <<
'\0';
488 Offset PrologueOffset = Out.tell();
489 for (
unsigned i = 0; i < 4; ++i)
493 if (!MainFile.empty()) {
508 const SrcMgr::ContentCache &
C = *I->second;
512 if (llvm::sys::path::is_relative(FE->
getName()))
515 const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(),
SM);
519 const llvm::MemoryBuffer *FromFile = SM.
getBuffer(FID);
520 Lexer L(FID, FromFile, SM, LOpts);
521 PM.insert(FE, LexTokens(L));
525 const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
528 Offset SpellingOff = EmitCachedSpellings();
531 Offset FileTableOff = EmitFileTable();
534 uint64_t Off = PrologueOffset;
550 StatListener(
PTHMap &pm) : PM(pm) {}
551 ~StatListener()
override {}
554 std::unique_ptr<vfs::File> *F,
558 if (Result == CacheMissing)
559 PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
562 if (llvm::sys::path::is_relative(Path))
565 PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry());
579 llvm::sys::fs::make_absolute(MainFilePath);
582 PTHWriter PW(*OS, PP);
585 auto StatCacheOwner = llvm::make_unique<StatListener>(PW.getPM());
586 StatListener *StatCache = StatCacheOwner.get();
598 PW.GeneratePTH(MainFilePath.str());
610 class PTHIdentifierTableTrait {
612 typedef PTHIdKey* key_type;
613 typedef key_type key_type_ref;
615 typedef uint32_t data_type;
616 typedef data_type data_type_ref;
618 typedef unsigned hash_value_type;
619 typedef unsigned offset_type;
621 static hash_value_type
ComputeHash(PTHIdKey* key) {
622 return llvm::HashString(key->II->getName());
625 static std::pair<unsigned,unsigned>
626 EmitKeyDataLength(raw_ostream& Out,
const PTHIdKey* key, uint32_t) {
627 using namespace llvm::support;
628 unsigned n = key->II->getLength() + 1;
629 endian::Writer<little>(Out).write<uint16_t>(n);
630 return std::make_pair(n,
sizeof(uint32_t));
633 static void EmitKey(raw_ostream& Out, PTHIdKey* key,
unsigned n) {
636 key->FileOffset = Out.tell();
637 Out.write(key->II->getNameStart(), n);
640 static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
642 using namespace llvm::support;
643 endian::Writer<little>(Out).write<uint32_t>(pID);
653 std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
659 PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount,
sizeof(PTHIdKey));
662 llvm::OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
668 assert(I->second > 0);
669 assert(I->second-1 < idcount);
670 unsigned idx = I->second-1;
673 IIDMap[idx].II = I->first;
676 IIOffMap.insert(&IIDMap[idx], I->second);
682 Offset StringTableOffset = IIOffMap.Emit(Out);
685 Offset IDOff = Out.tell();
687 for (
unsigned i = 0 ; i < idcount; ++i)
688 Emit32(IIDMap[i].FileOffset);
693 return std::make_pair(IDOff, StringTableOffset);
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
SourceManager & getSourceManager() const
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Defines the clang::FileManager interface and associated types.
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Defines the SourceManager interface.
Defines the FileSystemStatCache interface.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
fileinfo_iterator fileinfo_begin() const
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
Token - This structure provides full information about a lexed token.
static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off)
llvm::OnDiskChainedHashTableGenerator< FileEntryPTHEntryInfo > PTHMap
void setKind(tok::TokenKind K)
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
FileManager & getFileManager() const
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
Represents the results of name lookup.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS)
Cache tokens for use with PCH. Note that this requires a seekable stream.
tok::TokenKind getKind() const
detail::InMemoryDirectory::const_iterator I
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
Defines the clang::Preprocessor interface.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isNot(tok::TokenKind K) const
The result type of a method or function.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
static uint32_t swap32le(uint32_t X)
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
const char * getName() const
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
const TemplateArgument * iterator
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
FileID getMainFileID() const
Returns the FileID of the main source file.
unsigned ComputeHash(Selector Sel)
unsigned getFlags() const
Return the internal represtation of the flags.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
Represents a template argument.
void removeStatCache(FileSystemStatCache *statCache)
Removes the specified FileSystemStatCache object from the manager.
fileinfo_iterator fileinfo_end() const
detail::InMemoryDirectory::const_iterator E
llvm::DenseMap< const FileEntry *, SrcMgr::ContentCache * >::const_iterator fileinfo_iterator
Defines the Diagnostic-related interfaces.
void LexIncludeFilename(Token &Result)
After the preprocessor has parsed a #include, lex and (potentially) macro expand the filename...
raw_ostream & EmitString(raw_ostream &o, StringRef s)
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
static Decl::Kind getKind(const Decl *D)
unsigned getLength() const
Generate pre-tokenized header.
void clearFlag(TokenFlags Flag)
Unset the specified flag.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.