23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/Support/EndianStream.h"
26 #include "llvm/Support/MemoryBuffer.h"
28 #include <system_error>
29 using namespace clang;
40 PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
49 using namespace llvm::support;
52 const unsigned char *CurPtrShadow = CurPtr;
55 unsigned Word0 = endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
57 endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
59 endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
63 uint32_t Len = Word0 >> 16;
65 CurPtr = CurPtrShadow;
75 Tok.
setLocation(FileStartLoc.getLocWithOffset(FileOffset));
108 return LexEndOfFile(Tok);
119 if (TKind == tok::eod) {
144 diag::err_pp_unterminated_conditional);
161 "Must be in a preprocessing directive!");
170 const unsigned char* p = CurPtr;
189 using namespace llvm::support;
190 assert(CurPPCondPtr &&
"No cached PP conditional information.");
191 assert(LastHashTokPtr &&
"No known '#' token.");
193 const unsigned char *HashEntryI =
nullptr;
198 uint32_t
Offset = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
201 TableIdx = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
204 HashEntryI = TokBuf +
Offset;
210 if (HashEntryI < LastHashTokPtr && TableIdx) {
214 const unsigned char* NextPPCondPtr =
215 PPCond + TableIdx*(
sizeof(uint32_t)*2);
216 assert(NextPPCondPtr >= CurPPCondPtr);
218 const unsigned char *HashEntryJ =
219 TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
221 if (HashEntryJ <= LastHashTokPtr) {
223 HashEntryI = HashEntryJ;
224 TableIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
225 CurPPCondPtr = NextPPCondPtr;
229 while (HashEntryI < LastHashTokPtr);
230 assert(HashEntryI == LastHashTokPtr &&
"No PP-cond entry found for '#'");
231 assert(TableIdx &&
"No jumping from #endifs.");
234 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(
sizeof(uint32_t)*2);
235 assert(NextPPCondPtr >= CurPPCondPtr);
236 CurPPCondPtr = NextPPCondPtr;
240 TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
241 uint32_t NextIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
245 bool isEndif = NextIdx == 0;
256 if (CurPtr > HashEntryI) {
262 LastHashTokPtr = HashEntryI;
272 LastHashTokPtr = CurPtr;
292 using namespace llvm::support;
295 uint32_t
Offset = endian::readNext<uint32_t, little, aligned>(OffsetPtr);
308 const uint32_t TokenOff;
309 const uint32_t PPCondOff;
311 PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
312 : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
314 uint32_t getTokenOffset()
const {
return TokenOff; }
315 uint32_t getPPCondOffset()
const {
return PPCondOff; }
319 class PTHFileLookupCommonTrait {
321 typedef std::pair<unsigned char, const char*> internal_key_type;
322 typedef unsigned hash_value_type;
323 typedef unsigned offset_type;
325 static hash_value_type
ComputeHash(internal_key_type x) {
326 return llvm::HashString(x.second);
329 static std::pair<unsigned, unsigned>
330 ReadKeyDataLength(
const unsigned char*& d) {
331 using namespace llvm::support;
333 (
unsigned)endian::readNext<uint16_t, little, unaligned>(d);
334 unsigned dataLen = (
unsigned) *(d++);
335 return std::make_pair(keyLen, dataLen);
338 static internal_key_type ReadKey(
const unsigned char* d,
unsigned) {
339 unsigned char k = *(d++);
340 return std::make_pair(k, (
const char*) d);
352 return std::make_pair((
unsigned char) 0x1, FE->
getName());
355 static bool EqualKey(internal_key_type a, internal_key_type b) {
356 return a.first == b.first && strcmp(a.second, b.second) == 0;
359 static PTHFileData
ReadData(
const internal_key_type& k,
360 const unsigned char* d,
unsigned) {
361 assert(k.first == 0x1 &&
"Only file lookups can match!");
362 using namespace llvm::support;
363 uint32_t x = endian::readNext<uint32_t, little, unaligned>(d);
364 uint32_t y = endian::readNext<uint32_t, little, unaligned>(d);
365 return PTHFileData(x, y);
379 return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
384 return llvm::HashString(StringRef(a.first, a.second));
388 static const internal_key_type&
391 static std::pair<unsigned, unsigned>
393 using namespace llvm::support;
394 return std::make_pair(
395 (
unsigned)endian::readNext<uint16_t, little, unaligned>(d),
399 static std::pair<const char*, unsigned>
401 assert(n >= 2 && d[n-1] ==
'\0');
402 return std::make_pair((
const char*) d, n-1);
407 using namespace llvm::support;
408 return endian::readNext<uint32_t, little, unaligned>(d);
416 PTHManager::PTHManager(
417 std::unique_ptr<const llvm::MemoryBuffer> buf,
418 std::unique_ptr<PTHFileLookup> fileLookup,
const unsigned char *idDataTable,
419 std::unique_ptr<
IdentifierInfo *[], llvm::FreeDeleter> perIDCache,
420 std::unique_ptr<PTHStringIdLookup> stringIdLookup,
unsigned numIds,
421 const unsigned char *spellingBase,
const char *originalSourceFile)
422 : Buf(std::move(buf)), PerIDCache(std::move(perIDCache)),
423 FileLookup(std::move(fileLookup)), IdDataTable(idDataTable),
424 StringIdLookup(std::move(stringIdLookup)), NumIds(numIds),
PP(nullptr),
425 SpellingBase(spellingBase), OriginalSourceFile(originalSourceFile) {}
427 PTHManager::~PTHManager() {
436 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
437 llvm::MemoryBuffer::getFile(file);
441 Diags.
Report(diag::err_invalid_pth_file) << file;
444 std::unique_ptr<llvm::MemoryBuffer> File = std::move(FileOrErr.get());
446 using namespace llvm::support;
450 const unsigned char *BufBeg = (
const unsigned char*)File->getBufferStart();
451 const unsigned char *BufEnd = (
const unsigned char*)File->getBufferEnd();
454 if ((BufEnd - BufBeg) < (
signed)(
sizeof(
"cfe-pth") + 4 + 4) ||
455 memcmp(BufBeg,
"cfe-pth",
sizeof(
"cfe-pth")) != 0) {
456 Diags.
Report(diag::err_invalid_pth_file) << file;
461 const unsigned char *p = BufBeg + (
sizeof(
"cfe-pth"));
462 unsigned Version = endian::readNext<uint32_t, little, aligned>(p);
464 if (Version < PTHManager::Version) {
466 Version < PTHManager::Version
467 ?
"PTH file uses an older PTH format that is no longer supported"
468 :
"PTH file uses a newer PTH format that cannot be read");
473 const unsigned char *PrologueOffset = p;
475 if (PrologueOffset >= BufEnd) {
476 Diags.
Report(diag::err_invalid_pth_file) << file;
482 const unsigned char* FileTableOffset = PrologueOffset +
sizeof(uint32_t)*2;
483 const unsigned char *FileTable =
484 BufBeg + endian::readNext<uint32_t, little, aligned>(FileTableOffset);
486 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
487 Diags.
Report(diag::err_invalid_pth_file) << file;
496 InvalidPTH(Diags,
"PTH file contains no cached source data");
500 const unsigned char* IDTableOffset = PrologueOffset +
sizeof(uint32_t)*0;
501 const unsigned char *IData =
502 BufBeg + endian::readNext<uint32_t, little, aligned>(IDTableOffset);
504 if (!(IData >= BufBeg && IData < BufEnd)) {
505 Diags.
Report(diag::err_invalid_pth_file) << file;
511 const unsigned char* StringIdTableOffset = PrologueOffset +
sizeof(uint32_t)*1;
512 const unsigned char *StringIdTable =
513 BufBeg + endian::readNext<uint32_t, little, aligned>(StringIdTableOffset);
514 if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
515 Diags.
Report(diag::err_invalid_pth_file) << file;
519 std::unique_ptr<PTHStringIdLookup> SL(
523 const unsigned char* spellingBaseOffset = PrologueOffset +
sizeof(uint32_t)*3;
524 const unsigned char *spellingBase =
525 BufBeg + endian::readNext<uint32_t, little, aligned>(spellingBaseOffset);
526 if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
527 Diags.
Report(diag::err_invalid_pth_file) << file;
532 uint32_t NumIds = endian::readNext<uint32_t, little, aligned>(IData);
537 std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> PerIDCache;
540 PerIDCache.reset((
IdentifierInfo **)calloc(NumIds,
sizeof(PerIDCache[0])));
542 InvalidPTH(Diags,
"Could not allocate memory for processing PTH file");
548 const unsigned char* originalSourceBase = PrologueOffset +
sizeof(uint32_t)*4;
550 endian::readNext<uint16_t, little, unaligned>(originalSourceBase);
551 if (!len) originalSourceBase =
nullptr;
554 return new PTHManager(std::move(File), std::move(FL), IData,
555 std::move(PerIDCache), std::move(SL), NumIds,
556 spellingBase, (
const char *)originalSourceBase);
559 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(
unsigned PersistentID) {
560 using namespace llvm::support;
562 const unsigned char* TableEntry = IdDataTable +
sizeof(uint32_t)*PersistentID;
563 const unsigned char *IDData =
564 (
const unsigned char *)Buf->getBufferStart() +
565 endian::readNext<uint32_t, little, aligned>(TableEntry);
566 assert(IDData < (
const unsigned char*)Buf->getBufferEnd());
569 std::pair<IdentifierInfo,const unsigned char*> *Mem =
570 Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
572 Mem->second = IDData;
573 assert(IDData[0] !=
'\0');
577 PerIDCache[PersistentID] = II;
578 assert(II->getNameStart() && II->getNameStart()[0] !=
'\0');
584 assert(Name.empty() || Name.back() !=
'\0');
586 StringIdLookup->find(std::make_pair(Name.data(), Name.size()));
587 if (I == StringIdLookup->end())
592 return GetIdentifierInfo(*I-1);
596 const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
600 using namespace llvm::support;
607 if (
I == FileLookup->end())
612 const unsigned char *BufStart = (
const unsigned char *)Buf->getBufferStart();
614 const unsigned char* data = BufStart +
FileData.getTokenOffset();
617 const unsigned char* ppcond = BufStart +
FileData.getPPCondOffset();
618 uint32_t Len = endian::readNext<uint32_t, little, aligned>(ppcond);
619 if (Len == 0) ppcond =
nullptr;
621 assert(PP &&
"No preprocessor set yet!");
622 return new PTHLexer(*PP, FID, data, ppcond, *
this);
635 llvm::sys::fs::UniqueID UniqueID;
638 PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
640 : HasData(
true), Size(Size), ModTime(ModTime), UniqueID(UniqueID),
641 IsDirectory(IsDirectory) {}
643 PTHStatData() : HasData(
false) {}
646 class PTHStatLookupTrait :
public PTHFileLookupCommonTrait {
648 typedef const char* external_key_type;
649 typedef PTHStatData data_type;
651 static internal_key_type GetInternalKey(
const char *path) {
653 return std::make_pair((
unsigned char) 0x0, path);
656 static bool EqualKey(internal_key_type a, internal_key_type b) {
659 return strcmp(a.second, b.second) == 0;
662 static data_type ReadData(
const internal_key_type& k,
const unsigned char* d,
666 bool IsDirectory =
true;
667 if (k.first == 0x1 ) {
672 using namespace llvm::support;
674 uint64_t File = endian::readNext<uint64_t, little, unaligned>(d);
675 uint64_t Device = endian::readNext<uint64_t, little, unaligned>(d);
676 llvm::sys::fs::UniqueID UniqueID(Device, File);
677 time_t ModTime = endian::readNext<uint64_t, little, unaligned>(d);
678 uint64_t Size = endian::readNext<uint64_t, little, unaligned>(d);
679 return data_type(Size, ModTime, UniqueID, IsDirectory);
695 :
Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
699 std::unique_ptr<vfs::File> *F,
705 if (I ==
Cache.end())
706 return statChained(Path, Data, isFile, F, FS);
708 const PTHStatData &D = *
I;
726 std::unique_ptr<FileSystemStatCache> PTHManager::createStatCache() {
727 return llvm::make_unique<PTHStatCache>(*FileLookup);
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
SourceManager & getSourceManager() const
static const unsigned StoredTokenSize
LookupResult getStat(const char *Path, FileData &Data, bool isFile, std::unique_ptr< vfs::File > *F, vfs::FileSystem &FS) override
const FileEntry * external_key_type
Defines the clang::FileManager interface and associated types.
bool Lex(Token &Tok)
Lex - Return the next token.
static PTHFileData ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
static std::pair< unsigned, unsigned > ReadKeyDataLength(const unsigned char *&d)
Defines the FileSystemStatCache interface.
TypePropertyCache< Private > Cache
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setFlag(TokenFlags Flag)
Set the specified flag.
uint32_t IdentifierID
An ID number that refers to an identifier in an AST file.
static hash_value_type ComputeHash(const internal_key_type &a)
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
SmallVector< PPConditionalInfo, 4 > ConditionalStack
Information about the set of #if/#ifdef/#ifndef blocks we are currently in.
Token - This structure provides full information about a lexed token.
static uint32_t ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
void setKind(tok::TokenKind K)
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
static internal_key_type GetInternalKey(const FileEntry *FE)
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
detail::InMemoryDirectory::const_iterator I
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
bool ParsingFilename
True after #include; turns <xx> into a tok::angle_string_literal token.
bool LexingRawMode
True if in raw mode.
Defines the clang::Preprocessor interface.
external_key_type internal_key_type
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
void DiscardToEndOfLine()
DiscardToEndOfLine - Read the rest of the current preprocessor line as an uninterpreted string...
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
SourceLocation getSourceLocation() override
getSourceLocation - Return a source location for the token in the current file.
const char * getName() const
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
const TemplateArgument * iterator
void setLength(unsigned Len)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
static bool EqualKey(const internal_key_type &a, const internal_key_type &b)
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
unsigned ComputeHash(Selector Sel)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool SkipBlock()
SkipBlock - Used by Preprocessor to skip the current conditional block.
void setLiteralData(const char *Ptr)
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
PTHStatCache(PTHManager::PTHFileLookup &FL)
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Defines the clang::TokenKind enum and support functions.
const std::pair< const char *, unsigned > external_key_type
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
void setLocation(SourceLocation L)
static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg)
static const internal_key_type & GetInternalKey(const external_key_type &x)
static std::pair< const char *, unsigned > ReadKey(const unsigned char *d, unsigned n)
static bool EqualKey(internal_key_type a, internal_key_type b)
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.