20 #include "llvm/ADT/SmallString.h"
22 using namespace clang;
40 NextTokGetsSpace =
false;
43 DisableMacroExpansion =
false;
51 assert(Tokens[0].getLocation().isValid());
52 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
53 "Macro defined in macro?");
54 assert(ExpandLocStart.
isValid());
71 ExpandFunctionArguments();
82 bool disableMacroExpansion,
bool ownsTokens) {
90 OwnsTokens = ownsTokens;
91 DisableMacroExpansion = disableMacroExpansion;
95 AtStartOfLine =
false;
96 HasLeadingSpace =
false;
97 NextTokGetsSpace =
false;
108 void TokenLexer::destroy() {
118 if (ActualArgs) ActualArgs->
destroy(PP);
121 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
131 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
144 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
148 if (HasPasteOperator)
149 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
152 ResultToks.pop_back();
154 if (!ResultToks.empty()) {
159 if (ResultToks.back().is(tok::hashhash))
160 ResultToks.pop_back();
167 NextTokGetsSpace =
false;
173 void TokenLexer::ExpandFunctionArguments() {
179 bool MadeChange =
false;
181 for (
unsigned i = 0, e = NumTokens; i != e; ++i) {
185 const Token &CurTok = Tokens[i];
186 if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
187 NextTokGetsSpace =
true;
189 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
190 int ArgNo = Macro->
getArgumentNum(Tokens[i+1].getIdentifierInfo());
191 assert(ArgNo != -1 &&
"Token following # is not an argument?");
194 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
196 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
199 if (CurTok.
is(tok::hash))
214 if (NextTokGetsSpace)
217 ResultToks.push_back(Res);
220 NextTokGetsSpace =
false;
225 bool NonEmptyPasteBefore =
226 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
227 bool PasteBefore = i != 0 && Tokens[i-1].
is(tok::hashhash);
228 bool PasteAfter = i+1 != e && Tokens[i+1].
is(tok::hashhash);
229 assert(!NonEmptyPasteBefore || PasteBefore);
237 ResultToks.push_back(CurTok);
239 if (NextTokGetsSpace) {
241 NextTokGetsSpace =
false;
242 }
else if (PasteBefore && !NonEmptyPasteBefore)
257 MaybeRemoveCommaBeforeVaArgs(ResultToks,
265 if (!PasteBefore && !PasteAfter) {
266 const Token *ResultArgToks;
274 ResultArgToks = ArgTok;
278 unsigned FirstResult = ResultToks.size();
280 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
287 ResultToks.back().is(tok::comma))
292 for (
unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
293 Token &Tok = ResultToks[i];
294 if (Tok.
is(tok::hashhash))
300 ResultToks.begin()+FirstResult,
310 NextTokGetsSpace =
false;
324 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
325 ResultToks[ResultToks.size()-2].is(tok::comma) &&
329 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
332 ResultToks.append(ArgToks, ArgToks+NumToks);
336 for (
unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
338 Token &Tok = ResultToks[i];
339 if (Tok.
is(tok::hashhash))
343 if (ExpandLocStart.
isValid()) {
345 ResultToks.end()-NumToks, ResultToks.end());
358 if (NextTokGetsSpace)
361 NextTokGetsSpace =
false;
380 if (NonEmptyPasteBefore) {
381 assert(ResultToks.back().is(tok::hashhash));
382 ResultToks.pop_back();
390 MaybeRemoveCommaBeforeVaArgs(ResultToks,
397 assert(!OwnsTokens &&
"This would leak if we already own the token list");
399 NumTokens = ResultToks.size();
402 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
411 const Token &SecondTok) {
412 return FirstTok.
is(tok::identifier) &&
438 bool isFirstToken = CurToken == 0;
441 Tok = Tokens[CurToken++];
443 bool TokenIsFromPaste =
false;
447 if (!isAtEnd() && Macro &&
448 (Tokens[CurToken].is(tok::hashhash) ||
456 if (PasteTokens(Tok))
459 TokenIsFromPaste =
true;
467 if (ExpandLocStart.
isValid() &&
471 if (Tok.
is(tok::comment)) {
477 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
494 AtStartOfLine =
false;
495 HasLeadingSpace =
false;
523 bool TokenLexer::PasteTokens(
Token &Tok) {
527 if (PP.
getLangOpts().MicrosoftExt && (CurToken >= 2) &&
528 Tokens[CurToken - 2].is(tok::hashhash))
532 const char *ResultTokStrPtr =
nullptr;
538 if (Tokens[CurToken].is(tok::hashhash))
540 assert(!isAtEnd() &&
"No token on the RHS of a paste operator!");
543 const Token &RHS = Tokens[CurToken];
550 const char *BufPtr = &Buffer[0];
551 bool Invalid =
false;
552 unsigned LHSLen = PP.
getSpelling(Tok, BufPtr, &Invalid);
553 if (BufPtr != &Buffer[0])
554 memcpy(&Buffer[0], BufPtr, LHSLen);
558 BufPtr = Buffer.data() + LHSLen;
559 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
562 if (RHSLen && BufPtr != &Buffer[LHSLen])
564 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
567 Buffer.resize(LHSLen+RHSLen);
576 ResultTokTmp.
setKind(tok::string_literal);
589 Result.
setKind(tok::raw_identifier);
597 "Should be a raw location into scratch buffer");
599 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
601 bool Invalid =
false;
602 const char *ScratchBufStart
603 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
609 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
611 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
617 bool isInvalid = !TL.LexFromRawLexer(Result);
636 RHS.
is(tok::slash)) {
637 HandleMicrosoftCommentPaste(Tok, Loc);
647 : diag::err_pp_bad_paste)
657 if (Result.
is(tok::hashhash))
668 }
while (!isAtEnd() && Tokens[CurToken].
is(tok::hashhash));
679 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
681 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
683 while (SM.
getFileID(StartLoc) != MacroFID)
694 if (Tok.
is(tok::raw_identifier)) {
709 return Tokens[CurToken].
is(tok::l_paren);
715 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
724 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
731 assert(Macro &&
"Token streams can't paste comments");
742 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
743 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
744 "Not appropriate for token streams");
748 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
749 "Expected loc to come from the macro definition");
751 unsigned relativeOffset = 0;
767 Token *&begin_tokens,
768 Token * end_tokens) {
769 assert(begin_tokens < end_tokens);
784 Token *NextTok = begin_tokens + 1;
785 for (; NextTok < end_tokens; ++NextTok) {
795 if (RelOffs < 0 || RelOffs > 50)
806 Token &LastConsecutiveTok = *(NextTok-1);
810 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
818 for (; begin_tokens < NextTok; ++begin_tokens) {
819 Token &Tok = *begin_tokens;
832 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
838 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
840 while (begin_tokens < end_tokens) {
842 if (end_tokens - begin_tokens == 1) {
843 Token &Tok = *begin_tokens;
854 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &Result) {
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
SourceManager & getSourceManager() const
bool isPoisoned() const
Return true if this token has been poisoned.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
Init - Initialize this TokenLexer to expand from the specified macro with the specified argument info...
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
unsigned isNextTokenLParen() const
isNextTokenLParen - If the next token lexed will pop this macro off the expansion stack...
Defines the SourceManager interface.
unsigned getNextLocalOffset() const
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
void setRawIdentifierData(const char *Ptr)
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const std::vector< Token > & getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
tokens_iterator tokens_begin() const
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been 'stringified...
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
unsigned getNumArgs() const
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
Defines the clang::Preprocessor interface.
int getArgumentNum(const IdentifierInfo *Arg) const
Return the argument number of the specified identifier, or -1 if the identifier is not a formal argum...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
unsigned getDefinitionLength(SourceManager &SM) const
Get length in characters of the macro definition.
bool isNot(tok::TokenKind K) const
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
Encodes a location in the source.
void setLength(unsigned Len)
bool isValid() const
Return true if this is a valid SourceLocation object.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion, return false.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
static bool isInvalid(LocType Loc, bool *Invalid)
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool isFunctionLike() const
Encapsulates the data about a macro definition (e.g.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
tokens_iterator tokens_end() const
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
unsigned getLength() const
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const