15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/IntrusiveRefCntPtr.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/TinyPtrVector.h"
34 #include "llvm/Support/Allocator.h"
35 #include "llvm/Support/Registry.h"
40 template<
unsigned InternalLen>
class SmallString;
46 class ExternalPreprocessorSource;
50 class PragmaNamespace;
56 class CodeCompletionHandler;
57 class DirectoryLookup;
58 class PreprocessingRecord;
61 class PreprocessorOptions;
71 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
72 assert(Kind != tok::identifier &&
73 "Identifiers should be created by TokenValue(IdentifierInfo *)");
105 std::unique_ptr<ScratchBuffer> ScratchBuf;
115 std::unique_ptr<PTHManager> PTH;
119 llvm::BumpPtrAllocator BP;
145 unsigned CounterValue;
149 MaxAllowedIncludeStackDepth = 200
153 bool KeepComments : 1;
154 bool KeepMacroComments : 1;
155 bool SuppressIncludeNotFoundError : 1;
158 bool InMacroArgs : 1;
161 bool OwnsHeaderSearch : 1;
164 bool DisableMacroExpansion : 1;
168 bool MacroExpansionInDirectivesOverride : 1;
173 mutable bool ReadMacrosFromExternalSource : 1;
176 bool PragmasEnabled : 1;
179 bool PreprocessedOutput : 1;
182 bool ParsingIfOrElifDirective;
185 bool InMacroArgPreExpansion;
207 std::unique_ptr<PragmaNamespace> PragmaHandlers;
211 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
215 std::vector<CommentHandler *> CommentHandlers;
219 bool IncrementalProcessing;
231 unsigned CodeCompletionOffset;
255 bool ModuleImportExpectsIdentifier;
266 bool CodeCompletionReached;
278 std::pair<int, bool> SkipMainFilePreamble;
284 std::unique_ptr<Lexer> CurLexer;
290 std::unique_ptr<PTHLexer> CurPTHLexer;
308 std::unique_ptr<TokenLexer> CurTokenLexer;
316 CLK_LexAfterModuleImport
326 struct IncludeStackInfo {
327 enum CurLexerKind CurLexerKind;
329 std::unique_ptr<Lexer> TheLexer;
330 std::unique_ptr<PTHLexer> ThePTHLexer;
332 std::unique_ptr<TokenLexer> TheTokenLexer;
337 IncludeStackInfo(
enum CurLexerKind CurLexerKind,
Module *TheSubmodule,
338 std::unique_ptr<Lexer> &&TheLexer,
339 std::unique_ptr<PTHLexer> &&ThePTHLexer,
341 std::unique_ptr<TokenLexer> &&TheTokenLexer,
343 : CurLexerKind(std::move(CurLexerKind)),
344 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
345 ThePTHLexer(std::move(ThePTHLexer)),
346 ThePPLexer(std::move(ThePPLexer)),
347 TheTokenLexer(std::move(TheTokenLexer)),
348 TheDirLookup(std::move(TheDirLookup)) {}
349 IncludeStackInfo(IncludeStackInfo &&RHS)
350 : CurLexerKind(std::move(RHS.CurLexerKind)),
351 TheSubmodule(std::move(RHS.TheSubmodule)),
352 TheLexer(std::move(RHS.TheLexer)),
353 ThePTHLexer(std::move(RHS.ThePTHLexer)),
354 ThePPLexer(std::move(RHS.ThePPLexer)),
355 TheTokenLexer(std::move(RHS.TheTokenLexer)),
356 TheDirLookup(std::move(RHS.TheDirLookup)) {}
358 std::vector<IncludeStackInfo> IncludeMacroStack;
362 std::unique_ptr<PPCallbacks> Callbacks;
364 struct MacroExpandsInfo {
369 : Tok(Tok), MD(MD), Range(Range) { }
374 struct ModuleMacroInfo {
376 : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(
false) {}
381 llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
384 unsigned ActiveModuleMacrosGeneration;
388 llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
393 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *>
State;
402 !PP.CurSubmoduleState->VisibleModules.getGeneration())
405 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
412 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
413 Info->ActiveModuleMacrosGeneration)
414 PP.updateModuleMacroInfo(II, *Info);
419 MacroState() : MacroState(
nullptr) {}
421 MacroState(MacroState &&O) LLVM_NOEXCEPT :
State(O.State) {
424 MacroState &operator=(MacroState &&O) LLVM_NOEXCEPT {
431 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
432 Info->~ModuleMacroInfo();
436 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
441 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
448 auto *Info = getModuleInfo(PP, II);
449 return Info ? Info->IsAmbiguous :
false;
453 if (
auto *Info = getModuleInfo(PP, II))
454 return Info->ActiveModuleMacros;
461 if (
auto *Latest = getLatest())
462 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
467 if (
auto *Info = getModuleInfo(PP, II)) {
468 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
469 Info->ActiveModuleMacros.begin(),
470 Info->ActiveModuleMacros.end());
471 Info->ActiveModuleMacros.clear();
472 Info->IsAmbiguous =
false;
476 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
477 return Info->OverriddenMacros;
482 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
484 if (Overrides.empty())
490 Info->OverriddenMacros.clear();
491 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
492 Overrides.begin(), Overrides.end());
493 Info->ActiveModuleMacrosGeneration = 0;
502 typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
506 struct SubmoduleState;
509 struct BuildingSubmoduleInfo {
511 SubmoduleState *OuterSubmoduleState,
512 unsigned OuterPendingModuleMacroNames)
513 : M(M), ImportLoc(ImportLoc), OuterSubmoduleState(OuterSubmoduleState),
514 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
521 SubmoduleState *OuterSubmoduleState;
523 unsigned OuterPendingModuleMacroNames;
525 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
528 struct SubmoduleState {
532 VisibleModuleSet VisibleModules;
536 std::map<Module*, SubmoduleState> Submodules;
539 SubmoduleState NullSubmoduleState;
543 SubmoduleState *CurSubmoduleState;
546 llvm::FoldingSet<ModuleMacro> ModuleMacros;
553 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
565 typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
566 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
575 llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
578 unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
579 unsigned NumIf, NumElse, NumEndif;
580 unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
581 unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
582 unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
587 std::string Predefines;
594 enum { TokenLexerCacheSize = 8 };
595 unsigned NumCachedTokenLexers;
596 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
605 std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
626 CachedTokensTy::size_type CachedLexPos;
633 std::vector<CachedTokensTy::size_type> BacktrackPositions;
635 struct MacroInfoChain {
637 MacroInfoChain *
Next;
642 MacroInfoChain *MIChainHead;
644 struct DeserializedMacroInfoChain {
646 unsigned OwningModuleID;
648 DeserializedMacroInfoChain *
Next;
650 DeserializedMacroInfoChain *DeserialMIChainHead;
653 Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
654 DiagnosticsEngine &diags, LangOptions &opts,
655 SourceManager &
SM, HeaderSearch &Headers,
656 ModuleLoader &TheModuleLoader,
657 IdentifierInfoLookup *IILookup =
nullptr,
658 bool OwnsHeaderSearch =
false,
670 const TargetInfo *AuxTarget =
nullptr);
708 ExternalSource = Source;
712 return ExternalSource;
724 return ParsingIfOrElifDirective;
729 this->KeepComments = KeepComments | KeepMacroComments;
730 this->KeepMacroComments = KeepMacroComments;
739 SuppressIncludeNotFoundError = Suppress;
743 return SuppressIncludeNotFoundError;
749 PreprocessedOutput = IsPreprocessedOutput;
758 return CurPPLexer == L;
787 C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
788 std::move(Callbacks));
789 Callbacks = std::move(C);
807 auto I = Submodules.find(M);
808 if (
I == Submodules.end())
810 auto J =
I->second.Macros.find(II);
811 if (J ==
I->second.Macros.end())
813 auto *MD = J->second.getLatest();
814 return MD && MD->isDefined();
821 MacroState &
S = CurSubmoduleState->Macros[II];
822 auto *MD = S.getLatest();
823 while (MD && isa<VisibilityMacroDirective>(MD))
824 MD = MD->getPrevious();
826 S.getActiveModuleMacros(*
this, II),
827 S.isAmbiguous(*
this, II));
835 MacroState &
S = CurSubmoduleState->Macros[II];
837 if (
auto *MD = S.getLatest())
841 S.getActiveModuleMacros(*
this, II),
842 S.isAmbiguous(*
this, II));
852 if (!MD || MD->getDefinition().isUndefined())
866 return MD.getMacroInfo();
899 auto I = LeafModuleMacros.find(II);
900 if (
I != LeafModuleMacros.end())
912 llvm::iterator_range<macro_iterator>
913 macros(
bool IncludeExternalMacros =
true)
const {
914 return llvm::make_range(
macro_begin(IncludeExternalMacros),
935 return &Identifiers.
get(Name);
970 CodeComplete = &Handler;
980 CodeComplete =
nullptr;
1030 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1031 bool DisableMacroExpansion,
bool OwnsTokens);
1035 bool DisableMacroExpansion) {
1036 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true);
1039 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false);
1082 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1089 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1090 if (AllowMacroExpansion)
1095 AllowMacroExpansion);
1101 const char *DiagnosticTag,
1102 bool AllowMacroExpansion);
1112 while (Result.
getKind() == tok::comment);
1118 bool OldVal = DisableMacroExpansion;
1119 DisableMacroExpansion =
true;
1124 DisableMacroExpansion = OldVal;
1132 while (Result.
getKind() == tok::comment);
1142 DisableMacroExpansion =
true;
1143 MacroExpansionInDirectivesOverride =
true;
1157 return PeekAhead(N+1);
1167 "Should only be called when tokens are cached for backtracking");
1168 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1169 &&
"Should revert tokens up to the last backtrack position, not more");
1170 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1171 "Corrupted backtrack positions ?");
1180 EnterCachingLexMode();
1193 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1195 AnnotatePreviousCachedTokens(Tok);
1201 assert(CachedLexPos != 0);
1225 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1247 IncrementalProcessing = value;
1265 unsigned Line,
unsigned Column);
1281 return CodeCompletionFileLoc;
1291 CodeCompletionReached =
true;
1301 return PragmaARCCFCodeAuditedLoc;
1307 PragmaARCCFCodeAuditedLoc = Loc;
1315 return PragmaAssumeNonNullLoc;
1321 PragmaAssumeNonNullLoc = Loc;
1337 SkipMainFilePreamble.first = Bytes;
1338 SkipMainFilePreamble.second = StartOfLine;
1345 return Diags->
Report(Loc, DiagID);
1361 bool *invalid =
nullptr)
const {
1390 bool *Invalid =
nullptr)
const {
1400 bool *Invalid =
nullptr)
const;
1405 bool IgnoreWhiteSpace =
false) {
1413 bool *Invalid =
nullptr)
const {
1414 assert(Tok.
is(tok::numeric_constant) &&
1415 Tok.
getLength() == 1 &&
"Called on unsupported token");
1416 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
1497 unsigned Char)
const {
1507 ++NumFastTokenPaste;
1531 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1546 if(II->isPoisoned()) {
1557 *Ident___exception_code,
1558 *Ident_GetExceptionCode;
1561 *Ident___exception_info,
1562 *Ident_GetExceptionInfo;
1565 *Ident___abnormal_termination,
1566 *Ident_AbnormalTermination;
1568 const char *getCurLexerEndPos();
1629 unsigned SubModuleID);
1653 bool SkipCache =
false);
1686 bool *ShadowFlag =
nullptr);
1690 void PushIncludeMacroStack() {
1691 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
1692 IncludeMacroStack.emplace_back(
1693 CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
1694 CurPPLexer, std::move(CurTokenLexer), CurDirLookup);
1695 CurPPLexer =
nullptr;
1698 void PopIncludeMacroStack() {
1699 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1700 CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1701 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1702 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1703 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1704 CurSubmodule = IncludeMacroStack.back().TheSubmodule;
1705 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1706 IncludeMacroStack.pop_back();
1709 void PropagateLineStartLeadingSpaceInfo(
Token &
Result);
1711 void EnterSubmodule(Module *M, SourceLocation ImportLoc);
1712 void LeaveSubmodule();
1716 bool needModuleMacros()
const;
1720 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
1725 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1726 SourceLocation Loc);
1727 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1728 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1742 bool *ShadowFlag =
nullptr);
1748 bool ReadMacroDefinitionArgList(MacroInfo *MI,
Token& LastTok);
1758 void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1759 bool FoundNonSkipPortion,
bool FoundElse,
1760 SourceLocation ElseLoc = SourceLocation());
1763 void PTHSkipExcludedConditionalBlock();
1769 bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1773 void RegisterBuiltinPragmas();
1776 void RegisterBuiltinMacros();
1781 bool HandleMacroExpandedIdentifier(
Token &Tok,
const MacroDefinition &MD);
1788 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1789 ArrayRef<Token> tokens);
1790 void removeCachedMacroExpandedTokensOfLastLexer();
1796 bool isNextPPTokenLParen();
1800 MacroArgs *ReadFunctionLikeMacroArgs(
Token &MacroName, MacroInfo *MI,
1801 SourceLocation &ExpansionEnd);
1805 void ExpandBuiltinMacro(
Token &Tok);
1810 void Handle_Pragma(
Token &Tok);
1814 void HandleMicrosoft__pragma(
Token &Tok);
1818 void EnterSourceFileWithLexer(Lexer *TheLexer,
const DirectoryLookup *Dir);
1822 void EnterSourceFileWithPTH(PTHLexer *PL,
const DirectoryLookup *Dir);
1825 void setPredefinesFileID(FileID FID) {
1826 assert(PredefinesFileID.
isInvalid() &&
"PredefinesFileID already set!");
1827 PredefinesFileID = FID;
1832 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
1833 return L ? !L->isPragmaLexer() : P !=
nullptr;
1836 static bool IsFileLexer(
const IncludeStackInfo&
I) {
1837 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1840 bool IsFileLexer()
const {
1841 return IsFileLexer(CurLexer.get(), CurPPLexer);
1847 bool InCachingLexMode()
const {
1850 return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1851 !IncludeMacroStack.empty();
1853 void EnterCachingLexMode();
1854 void ExitCachingLexMode() {
1855 if (InCachingLexMode())
1858 const Token &PeekAhead(
unsigned N);
1859 void AnnotatePreviousCachedTokens(
const Token &Tok);
1865 void HandleLineDirective(
Token &Tok);
1866 void HandleDigitDirective(
Token &Tok);
1867 void HandleUserDiagnosticDirective(
Token &Tok,
bool isWarning);
1868 void HandleIdentSCCSDirective(
Token &Tok);
1869 void HandleMacroPublicDirective(
Token &Tok);
1870 void HandleMacroPrivateDirective(
Token &Tok);
1873 void HandleIncludeDirective(SourceLocation HashLoc,
1875 const DirectoryLookup *LookupFrom =
nullptr,
1876 const FileEntry *LookupFromFile =
nullptr,
1877 bool isImport =
false);
1878 void HandleIncludeNextDirective(SourceLocation HashLoc,
Token &Tok);
1879 void HandleIncludeMacrosDirective(SourceLocation HashLoc,
Token &Tok);
1880 void HandleImportDirective(SourceLocation HashLoc,
Token &Tok);
1881 void HandleMicrosoftImportDirective(
Token &Tok);
1905 SourceLocation MLoc);
1909 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterTopLevelIfndef);
1910 void HandleUndefDirective(
Token &Tok);
1913 void HandleIfdefDirective(
Token &Tok,
bool isIfndef,
1914 bool ReadAnyTokensBeforeDirective);
1915 void HandleIfDirective(
Token &Tok,
bool ReadAnyTokensBeforeDirective);
1916 void HandleEndifDirective(
Token &Tok);
1917 void HandleElseDirective(
Token &Tok);
1918 void HandleElifDirective(
Token &Tok);
1921 void HandlePragmaDirective(SourceLocation IntroducerLoc,
1959 extern template class llvm::Registry<clang::PragmaHandler>;
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
SourceManager & getSourceManager() const
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer...
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ExternalPreprocessorSource * getExternalSource() const
void FinalizeForModelFile()
Cleanup after model file parsing.
SelectorTable & getSelectorTable()
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
bool ConcatenateIncludeName(SmallString< 128 > &FilenameBuffer, SourceLocation &End)
Handle cases where the #include name is expanded from a macro as multiple tokens, which need to be gl...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Implements support for file system lookup, file system caching, and directory search management...
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
Module * getCurrentSubmodule() const
Return the submodule owning the file being lexed.
friend void TokenLexer::ExpandFunctionArguments()
void MaybeHandlePoisonedIdentifier(Token &Identifier)
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
static const Builtin::Info BuiltinInfo[]
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void dumpMacroInfo(const IdentifierInfo *II)
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
const std::string & getPredefines() const
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
std::unique_ptr< llvm::MemoryBuffer > Buffer
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
MacroMap::const_iterator macro_iterator
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
unsigned getCounterValue() const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void setPTHManager(PTHManager *pm)
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion...
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Builtin::Context & getBuiltinInfo()
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
This interface provides a way to observe the actions of the preprocessor as it does its thing...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool getPragmasEnabled() const
TokenValue(tok::TokenKind Kind)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
Module * getModuleContainingLocation(SourceLocation Loc)
Find the module that contains the specified location, either directly or indirectly.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
A directive for a defined macro or a macro imported from a module.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
This table allows us to fully hide how we implement multi-keyword caching.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Describes a module or submodule.
FileManager & getFileManager() const
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void SetSuppressIncludeNotFoundError(bool Suppress)
void CheckEndOfDirective(const char *Directive, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
MacroUse
Context in which macro name is used.
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
bool getCommentRetentionState() const
const IdentifierTable & getIdentifierTable() const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
void LexNonComment(Token &Result)
Lex a token.
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
HeaderSearch & getHeaderSearchInfo() const
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
bool hadModuleLoaderFatalFailure() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
tok::TokenKind getKind() const
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
const TargetInfo & getTargetInfo() const
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Character, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token...
detail::InMemoryDirectory::const_iterator I
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
A little helper class used to produce diagnostics.
void HandlePragmaIncludeAlias(Token &Tok)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once.
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Exposes information about the current target.
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Abstract interface for external sources of preprocessor information.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
void makeModuleVisible(Module *M, SourceLocation Loc)
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Implements an efficient mapping from strings to IdentifierInfo nodes.
bool LexOnOffSwitch(tok::OnOffSwitch &OOS)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
bool operator==(const Token &Tok) const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
TokenValue(IdentifierInfo *II)
SourceLocation getModuleImportLoc(Module *M) const
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
The result type of a method or function.
Preprocessor(IntrusiveRefCntPtr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
MacroInfo * AllocateDeserializedMacroInfo(SourceLocation L, unsigned SubModuleID)
Allocate a new MacroInfo object loaded from an AST file.
size_t getTotalMemory() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
macro_iterator macro_end(bool IncludeExternalMacros=true) const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierTable & getIdentifierTable()
void setPragmasEnabled(bool Enabled)
Cached information about one file (either on disk or in the virtual file system). ...
const TargetInfo * getAuxTargetInfo() const
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
void Lex(Token &Result)
Lex the next token for this preprocessor.
ArrayRef< FormatToken * > Tokens
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
DiagnosticsEngine & getDiagnostics() const
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Reads an AST files chain containing the contents of a translation unit.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Abstract interface for a module loader.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Defines the Diagnostic-related interfaces.
void PoisonSEHIdentifiers(bool Poison=true)
Encapsulates the data about a macro definition (e.g.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Cached information about one directory (either on disk or in the virtual file system).
Defines the PPCallbacks interface.
const FileEntry * getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning a missing module import.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool HandleComment(Token &Token, SourceRange Comment)
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setSuppressAllDiagnostics(bool Val=true)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
void DumpMacro(const MacroInfo &MI) const
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
ModuleMacro * getModuleMacro(Module *Mod, IdentifierInfo *II)
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
void HandlePragmaPoison(Token &PoisonTok)
HandlePragmaPoison - Handle #pragma GCC poison.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
TranslationUnitKind
Describes the kind of translation unit being processed.
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
The translation unit is a complete translation unit.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
void DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found...
unsigned getLength() const
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
void DumpLocation(SourceLocation Loc) const
void setCounterValue(unsigned V)
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
PTHManager * getPTHManager()
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
const DefMacroDirective * getDirective() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
void setDiagnostics(DiagnosticsEngine &D)
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const