17 #include "llvm/Support/ErrorHandling.h"
18 using namespace clang;
26 (CPlusPlus11 && (Str[0] ==
'u' || Str[0] ==
'U' || Str[0] ==
'R'))) {
33 if (Str[1] ==
'R' && Str[0] !=
'R' && Str.size() == 2 &&
CPlusPlus11)
37 if (Str[0] ==
'u' && Str[1] ==
'8') {
38 if (Str.size() == 2)
return true;
39 if (Str.size() == 3 && Str[2] ==
'R')
return true;
48 bool TokenConcatenation::IsIdentifierStringPrefix(
const Token &Tok)
const {
57 LangOpts.CPlusPlus11);
62 const char *TokPtr =
Buffer;
64 return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus11);
71 memset(TokenInfo, 0,
sizeof(TokenInfo));
74 TokenInfo[tok::identifier ] |= aci_custom;
75 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;
76 TokenInfo[tok::period ] |= aci_custom_firstchar;
77 TokenInfo[tok::amp ] |= aci_custom_firstchar;
78 TokenInfo[tok::plus ] |= aci_custom_firstchar;
79 TokenInfo[tok::minus ] |= aci_custom_firstchar;
80 TokenInfo[tok::slash ] |= aci_custom_firstchar;
81 TokenInfo[tok::less ] |= aci_custom_firstchar;
82 TokenInfo[tok::greater ] |= aci_custom_firstchar;
83 TokenInfo[tok::pipe ] |= aci_custom_firstchar;
84 TokenInfo[tok::percent ] |= aci_custom_firstchar;
85 TokenInfo[tok::colon ] |= aci_custom_firstchar;
86 TokenInfo[tok::hash ] |= aci_custom_firstchar;
87 TokenInfo[tok::arrow ] |= aci_custom_firstchar;
91 TokenInfo[tok::string_literal ] |= aci_custom;
92 TokenInfo[tok::wide_string_literal ] |= aci_custom;
93 TokenInfo[tok::utf8_string_literal ] |= aci_custom;
94 TokenInfo[tok::utf16_string_literal] |= aci_custom;
95 TokenInfo[tok::utf32_string_literal] |= aci_custom;
96 TokenInfo[tok::char_constant ] |= aci_custom;
97 TokenInfo[tok::wide_char_constant ] |= aci_custom;
98 TokenInfo[tok::utf16_char_constant ] |= aci_custom;
99 TokenInfo[tok::utf32_char_constant ] |= aci_custom;
104 TokenInfo[tok::utf8_char_constant] |= aci_custom;
107 TokenInfo[tok::amp ] |= aci_avoid_equal;
108 TokenInfo[tok::plus ] |= aci_avoid_equal;
109 TokenInfo[tok::minus ] |= aci_avoid_equal;
110 TokenInfo[tok::slash ] |= aci_avoid_equal;
111 TokenInfo[tok::less ] |= aci_avoid_equal;
112 TokenInfo[tok::greater ] |= aci_avoid_equal;
113 TokenInfo[tok::pipe ] |= aci_avoid_equal;
114 TokenInfo[tok::percent ] |= aci_avoid_equal;
115 TokenInfo[tok::star ] |= aci_avoid_equal;
116 TokenInfo[tok::exclaim ] |= aci_avoid_equal;
117 TokenInfo[tok::lessless ] |= aci_avoid_equal;
118 TokenInfo[tok::greatergreater] |= aci_avoid_equal;
119 TokenInfo[tok::caret ] |= aci_avoid_equal;
120 TokenInfo[tok::equal ] |= aci_avoid_equal;
128 return II->getNameStart()[0];
138 const char *TokPtr =
Buffer;
158 const Token &PrevTok,
159 const Token &Tok)
const {
171 PrevKind = tok::identifier;
174 unsigned ConcatInfo = TokenInfo[PrevKind];
177 if (ConcatInfo == 0)
return false;
179 if (ConcatInfo & aci_avoid_equal) {
181 if (Tok.
isOneOf(tok::equal, tok::equalequal))
183 ConcatInfo &= ~aci_avoid_equal;
187 assert(Tok.
isOneOf(tok::annot_module_include, tok::annot_module_begin,
188 tok::annot_module_end) &&
189 "unexpected annotation in AvoidConcat");
200 if (ConcatInfo & aci_custom) {
208 llvm_unreachable(
"InitAvoidConcatTokenInfo built wrong");
210 case tok::raw_identifier:
211 llvm_unreachable(
"tok::raw_identifier in non-raw lexing mode!");
213 case tok::string_literal:
214 case tok::wide_string_literal:
215 case tok::utf8_string_literal:
216 case tok::utf16_string_literal:
217 case tok::utf32_string_literal:
218 case tok::char_constant:
219 case tok::wide_char_constant:
220 case tok::utf8_char_constant:
221 case tok::utf16_char_constant:
222 case tok::utf32_char_constant:
236 case tok::identifier:
238 if (Tok.
is(tok::numeric_constant))
242 Tok.
isOneOf(tok::wide_string_literal, tok::utf8_string_literal,
243 tok::utf16_string_literal, tok::utf32_string_literal,
244 tok::wide_char_constant, tok::utf8_char_constant,
245 tok::utf16_char_constant, tok::utf32_char_constant))
249 if (Tok.
isNot(tok::char_constant) && Tok.
isNot(tok::string_literal))
254 return IsIdentifierStringPrefix(PrevTok);
256 case tok::numeric_constant:
258 FirstChar ==
'+' || FirstChar ==
'-';
260 return (FirstChar ==
'.' && PrevPrevTok.
is(tok::period)) ||
264 return FirstChar ==
'&';
266 return FirstChar ==
'+';
268 return FirstChar ==
'-' || FirstChar ==
'>';
270 return FirstChar ==
'*' || FirstChar ==
'/';
272 return FirstChar ==
'<' || FirstChar ==
':' || FirstChar ==
'%';
274 return FirstChar ==
'>';
276 return FirstChar ==
'|';
278 return FirstChar ==
'>' || FirstChar ==
':';
280 return FirstChar ==
'>' ||
283 return FirstChar ==
'#' || FirstChar ==
'@' || FirstChar ==
'%';
285 return PP.
getLangOpts().CPlusPlus && FirstChar ==
'*';
SourceManager & getSourceManager() const
static LLVM_READONLY bool isDigit(unsigned char c)
Return true if this character is an ASCII digit: [0-9].
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok, const Token &Tok) const
AvoidConcat - If printing PrevTok immediately followed by Tok would cause the two individual tokens t...
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
std::unique_ptr< llvm::MemoryBuffer > Buffer
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
static LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
One of these records is kept for each identifier that is lexed.
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getKind() const
Defines the clang::Preprocessor interface.
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
bool isNot(tok::TokenKind K) const
float __ovld __cnfn length(float p)
Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
TokenConcatenation(Preprocessor &PP)
Encodes a location in the source.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
static bool IsStringPrefix(StringRef Str, bool CPlusPlus11)
IsStringPrefix - Return true if Str is a string prefix.
static char GetFirstChar(Preprocessor &PP, const Token &Tok)
GetFirstChar - Get the first character of the token.
unsigned getLength() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const