LLVM 17.0.0git
TGLexer.cpp
Go to the documentation of this file.
1//===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implement the Lexer for TableGen.
10//
11//===----------------------------------------------------------------------===//
12
13#include "TGLexer.h"
14#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/Twine.h"
17#include "llvm/Config/config.h" // for strtoull()/strtoll() define
21#include "llvm/TableGen/Error.h"
22#include <algorithm>
23#include <cctype>
24#include <cerrno>
25#include <cstdint>
26#include <cstdio>
27#include <cstdlib>
28#include <cstring>
29
30using namespace llvm;
31
32namespace {
33// A list of supported preprocessing directives with their
34// internal token kinds and names.
35struct {
37 const char *Word;
38} PreprocessorDirs[] = {
39 { tgtok::Ifdef, "ifdef" },
40 { tgtok::Ifndef, "ifndef" },
41 { tgtok::Else, "else" },
42 { tgtok::Endif, "endif" },
43 { tgtok::Define, "define" }
44};
45} // end anonymous namespace
46
48 CurBuffer = SrcMgr.getMainFileID();
49 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
50 CurPtr = CurBuf.begin();
51 TokStart = nullptr;
52
53 // Pretend that we enter the "top-level" include file.
54 PrepIncludeStack.push_back(
55 std::make_unique<std::vector<PreprocessorControlDesc>>());
56
57 // Put all macros defined in the command line into the DefinedMacros set.
58 for (const std::string &MacroName : Macros)
59 DefinedMacros.insert(MacroName);
60}
61
63 return SMLoc::getFromPointer(TokStart);
64}
65
67 return {getLoc(), SMLoc::getFromPointer(CurPtr)};
68}
69
70/// ReturnError - Set the error to the specified string at the specified
71/// location. This is defined to always return tgtok::Error.
72tgtok::TokKind TGLexer::ReturnError(SMLoc Loc, const Twine &Msg) {
73 PrintError(Loc, Msg);
74 return tgtok::Error;
75}
76
77tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {
78 return ReturnError(SMLoc::getFromPointer(Loc), Msg);
79}
80
81bool TGLexer::processEOF() {
82 SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
83 if (ParentIncludeLoc != SMLoc()) {
84 // If prepExitInclude() detects a problem with the preprocessing
85 // control stack, it will return false. Pretend that we reached
86 // the final EOF and stop lexing more tokens by returning false
87 // to LexToken().
88 if (!prepExitInclude(false))
89 return false;
90
91 CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
92 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
93 CurPtr = ParentIncludeLoc.getPointer();
94 // Make sure TokStart points into the parent file's buffer.
95 // LexToken() assigns to it before calling getNextChar(),
96 // so it is pointing into the included file now.
97 TokStart = CurPtr;
98 return true;
99 }
100
101 // Pretend that we exit the "top-level" include file.
102 // Note that in case of an error (e.g. control stack imbalance)
103 // the routine will issue a fatal error.
104 prepExitInclude(true);
105 return false;
106}
107
108int TGLexer::getNextChar() {
109 char CurChar = *CurPtr++;
110 switch (CurChar) {
111 default:
112 return (unsigned char)CurChar;
113
114 case 0: {
115 // A NUL character in the stream is either the end of the current buffer or
116 // a spurious NUL in the file. Disambiguate that here.
117 if (CurPtr - 1 == CurBuf.end()) {
118 --CurPtr; // Arrange for another call to return EOF again.
119 return EOF;
120 }
122 "NUL character is invalid in source; treated as space");
123 return ' ';
124 }
125
126 case '\n':
127 case '\r':
128 // Handle the newline character by ignoring it and incrementing the line
129 // count. However, be careful about 'dos style' files with \n\r in them.
130 // Only treat a \n\r or \r\n as a single line.
131 if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
132 *CurPtr != CurChar)
133 ++CurPtr; // Eat the two char newline sequence.
134 return '\n';
135 }
136}
137
138int TGLexer::peekNextChar(int Index) const {
139 return *(CurPtr + Index);
140}
141
142tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
143 TokStart = CurPtr;
144 // This always consumes at least one character.
145 int CurChar = getNextChar();
146
147 switch (CurChar) {
148 default:
149 // Handle letters: [a-zA-Z_]
150 if (isalpha(CurChar) || CurChar == '_')
151 return LexIdentifier();
152
153 // Unknown character, emit an error.
154 return ReturnError(TokStart, "Unexpected character");
155 case EOF:
156 // Lex next token, if we just left an include file.
157 // Note that leaving an include file means that the next
158 // symbol is located at the end of the 'include "..."'
159 // construct, so LexToken() is called with default
160 // false parameter.
161 if (processEOF())
162 return LexToken();
163
164 // Return EOF denoting the end of lexing.
165 return tgtok::Eof;
166
167 case ':': return tgtok::colon;
168 case ';': return tgtok::semi;
169 case ',': return tgtok::comma;
170 case '<': return tgtok::less;
171 case '>': return tgtok::greater;
172 case ']': return tgtok::r_square;
173 case '{': return tgtok::l_brace;
174 case '}': return tgtok::r_brace;
175 case '(': return tgtok::l_paren;
176 case ')': return tgtok::r_paren;
177 case '=': return tgtok::equal;
178 case '?': return tgtok::question;
179 case '#':
180 if (FileOrLineStart) {
181 tgtok::TokKind Kind = prepIsDirective();
182 if (Kind != tgtok::Error)
183 return lexPreprocessor(Kind);
184 }
185
186 return tgtok::paste;
187
188 // The period is a separate case so we can recognize the "..."
189 // range punctuator.
190 case '.':
191 if (peekNextChar(0) == '.') {
192 ++CurPtr; // Eat second dot.
193 if (peekNextChar(0) == '.') {
194 ++CurPtr; // Eat third dot.
195 return tgtok::dotdotdot;
196 }
197 return ReturnError(TokStart, "Invalid '..' punctuation");
198 }
199 return tgtok::dot;
200
201 case '\r':
202 PrintFatalError("getNextChar() must never return '\r'");
203 return tgtok::Error;
204
205 case ' ':
206 case '\t':
207 // Ignore whitespace.
208 return LexToken(FileOrLineStart);
209 case '\n':
210 // Ignore whitespace, and identify the new line.
211 return LexToken(true);
212 case '/':
213 // If this is the start of a // comment, skip until the end of the line or
214 // the end of the buffer.
215 if (*CurPtr == '/')
216 SkipBCPLComment();
217 else if (*CurPtr == '*') {
218 if (SkipCComment())
219 return tgtok::Error;
220 } else // Otherwise, this is an error.
221 return ReturnError(TokStart, "Unexpected character");
222 return LexToken(FileOrLineStart);
223 case '-': case '+':
224 case '0': case '1': case '2': case '3': case '4': case '5': case '6':
225 case '7': case '8': case '9': {
226 int NextChar = 0;
227 if (isdigit(CurChar)) {
228 // Allow identifiers to start with a number if it is followed by
229 // an identifier. This can happen with paste operations like
230 // foo#8i.
231 int i = 0;
232 do {
233 NextChar = peekNextChar(i++);
234 } while (isdigit(NextChar));
235
236 if (NextChar == 'x' || NextChar == 'b') {
237 // If this is [0-9]b[01] or [0-9]x[0-9A-fa-f] this is most
238 // likely a number.
239 int NextNextChar = peekNextChar(i);
240 switch (NextNextChar) {
241 default:
242 break;
243 case '0': case '1':
244 if (NextChar == 'b')
245 return LexNumber();
246 [[fallthrough]];
247 case '2': case '3': case '4': case '5':
248 case '6': case '7': case '8': case '9':
249 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
250 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
251 if (NextChar == 'x')
252 return LexNumber();
253 break;
254 }
255 }
256 }
257
258 if (isalpha(NextChar) || NextChar == '_')
259 return LexIdentifier();
260
261 return LexNumber();
262 }
263 case '"': return LexString();
264 case '$': return LexVarName();
265 case '[': return LexBracket();
266 case '!': return LexExclaim();
267 }
268}
269
270/// LexString - Lex "[^"]*"
271tgtok::TokKind TGLexer::LexString() {
272 const char *StrStart = CurPtr;
273
274 CurStrVal = "";
275
276 while (*CurPtr != '"') {
277 // If we hit the end of the buffer, report an error.
278 if (*CurPtr == 0 && CurPtr == CurBuf.end())
279 return ReturnError(StrStart, "End of file in string literal");
280
281 if (*CurPtr == '\n' || *CurPtr == '\r')
282 return ReturnError(StrStart, "End of line in string literal");
283
284 if (*CurPtr != '\\') {
285 CurStrVal += *CurPtr++;
286 continue;
287 }
288
289 ++CurPtr;
290
291 switch (*CurPtr) {
292 case '\\': case '\'': case '"':
293 // These turn into their literal character.
294 CurStrVal += *CurPtr++;
295 break;
296 case 't':
297 CurStrVal += '\t';
298 ++CurPtr;
299 break;
300 case 'n':
301 CurStrVal += '\n';
302 ++CurPtr;
303 break;
304
305 case '\n':
306 case '\r':
307 return ReturnError(CurPtr, "escaped newlines not supported in tblgen");
308
309 // If we hit the end of the buffer, report an error.
310 case '\0':
311 if (CurPtr == CurBuf.end())
312 return ReturnError(StrStart, "End of file in string literal");
313 [[fallthrough]];
314 default:
315 return ReturnError(CurPtr, "invalid escape in string literal");
316 }
317 }
318
319 ++CurPtr;
320 return tgtok::StrVal;
321}
322
323tgtok::TokKind TGLexer::LexVarName() {
324 if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
325 return ReturnError(TokStart, "Invalid variable name");
326
327 // Otherwise, we're ok, consume the rest of the characters.
328 const char *VarNameStart = CurPtr++;
329
330 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
331 ++CurPtr;
332
333 CurStrVal.assign(VarNameStart, CurPtr);
334 return tgtok::VarName;
335}
336
337tgtok::TokKind TGLexer::LexIdentifier() {
338 // The first letter is [a-zA-Z_].
339 const char *IdentStart = TokStart;
340
341 // Match the rest of the identifier regex: [0-9a-zA-Z_]*
342 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
343 ++CurPtr;
344
345 // Check to see if this identifier is a reserved keyword.
346 StringRef Str(IdentStart, CurPtr-IdentStart);
347
349 .Case("int", tgtok::Int)
350 .Case("bit", tgtok::Bit)
351 .Case("bits", tgtok::Bits)
352 .Case("string", tgtok::String)
353 .Case("list", tgtok::List)
354 .Case("code", tgtok::Code)
355 .Case("dag", tgtok::Dag)
356 .Case("class", tgtok::Class)
357 .Case("def", tgtok::Def)
358 .Case("true", tgtok::TrueVal)
359 .Case("false", tgtok::FalseVal)
360 .Case("foreach", tgtok::Foreach)
361 .Case("defm", tgtok::Defm)
362 .Case("defset", tgtok::Defset)
363 .Case("multiclass", tgtok::MultiClass)
364 .Case("field", tgtok::Field)
365 .Case("let", tgtok::Let)
366 .Case("in", tgtok::In)
367 .Case("defvar", tgtok::Defvar)
368 .Case("include", tgtok::Include)
369 .Case("if", tgtok::If)
370 .Case("then", tgtok::Then)
371 .Case("else", tgtok::ElseKW)
372 .Case("assert", tgtok::Assert)
374
375 // A couple of tokens require special processing.
376 switch (Kind) {
377 case tgtok::Include:
378 if (LexInclude()) return tgtok::Error;
379 return Lex();
380 case tgtok::Id:
381 CurStrVal.assign(Str.begin(), Str.end());
382 break;
383 default:
384 break;
385 }
386
387 return Kind;
388}
389
390/// LexInclude - We just read the "include" token. Get the string token that
391/// comes next and enter the include.
392bool TGLexer::LexInclude() {
393 // The token after the include must be a string.
394 tgtok::TokKind Tok = LexToken();
395 if (Tok == tgtok::Error) return true;
396 if (Tok != tgtok::StrVal) {
397 PrintError(getLoc(), "Expected filename after include");
398 return true;
399 }
400
401 // Get the string.
402 std::string Filename = CurStrVal;
403 std::string IncludedFile;
404
405 CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
406 IncludedFile);
407 if (!CurBuffer) {
408 PrintError(getLoc(), "Could not find include file '" + Filename + "'");
409 return true;
410 }
411
412 Dependencies.insert(IncludedFile);
413 // Save the line number and lex buffer of the includer.
414 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
415 CurPtr = CurBuf.begin();
416
417 PrepIncludeStack.push_back(
418 std::make_unique<std::vector<PreprocessorControlDesc>>());
419 return false;
420}
421
422/// SkipBCPLComment - Skip over the comment by finding the next CR or LF.
423/// Or we may end up at the end of the buffer.
424void TGLexer::SkipBCPLComment() {
425 ++CurPtr; // skip the second slash.
426 auto EOLPos = CurBuf.find_first_of("\r\n", CurPtr - CurBuf.data());
427 CurPtr = (EOLPos == StringRef::npos) ? CurBuf.end() : CurBuf.data() + EOLPos;
428}
429
430/// SkipCComment - This skips C-style /**/ comments. The only difference from C
431/// is that we allow nesting.
432bool TGLexer::SkipCComment() {
433 ++CurPtr; // skip the star.
434 unsigned CommentDepth = 1;
435
436 while (true) {
437 int CurChar = getNextChar();
438 switch (CurChar) {
439 case EOF:
440 PrintError(TokStart, "Unterminated comment!");
441 return true;
442 case '*':
443 // End of the comment?
444 if (CurPtr[0] != '/') break;
445
446 ++CurPtr; // End the */.
447 if (--CommentDepth == 0)
448 return false;
449 break;
450 case '/':
451 // Start of a nested comment?
452 if (CurPtr[0] != '*') break;
453 ++CurPtr;
454 ++CommentDepth;
455 break;
456 }
457 }
458}
459
460/// LexNumber - Lex:
461/// [-+]?[0-9]+
462/// 0x[0-9a-fA-F]+
463/// 0b[01]+
464tgtok::TokKind TGLexer::LexNumber() {
465 if (CurPtr[-1] == '0') {
466 if (CurPtr[0] == 'x') {
467 ++CurPtr;
468 const char *NumStart = CurPtr;
469 while (isxdigit(CurPtr[0]))
470 ++CurPtr;
471
472 // Requires at least one hex digit.
473 if (CurPtr == NumStart)
474 return ReturnError(TokStart, "Invalid hexadecimal number");
475
476 errno = 0;
477 CurIntVal = strtoll(NumStart, nullptr, 16);
478 if (errno == EINVAL)
479 return ReturnError(TokStart, "Invalid hexadecimal number");
480 if (errno == ERANGE) {
481 errno = 0;
482 CurIntVal = (int64_t)strtoull(NumStart, nullptr, 16);
483 if (errno == EINVAL)
484 return ReturnError(TokStart, "Invalid hexadecimal number");
485 if (errno == ERANGE)
486 return ReturnError(TokStart, "Hexadecimal number out of range");
487 }
488 return tgtok::IntVal;
489 } else if (CurPtr[0] == 'b') {
490 ++CurPtr;
491 const char *NumStart = CurPtr;
492 while (CurPtr[0] == '0' || CurPtr[0] == '1')
493 ++CurPtr;
494
495 // Requires at least one binary digit.
496 if (CurPtr == NumStart)
497 return ReturnError(CurPtr-2, "Invalid binary number");
498 CurIntVal = strtoll(NumStart, nullptr, 2);
499 return tgtok::BinaryIntVal;
500 }
501 }
502
503 // Check for a sign without a digit.
504 if (!isdigit(CurPtr[0])) {
505 if (CurPtr[-1] == '-')
506 return tgtok::minus;
507 else if (CurPtr[-1] == '+')
508 return tgtok::plus;
509 }
510
511 while (isdigit(CurPtr[0]))
512 ++CurPtr;
513 CurIntVal = strtoll(TokStart, nullptr, 10);
514 return tgtok::IntVal;
515}
516
517/// LexBracket - We just read '['. If this is a code block, return it,
518/// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
519tgtok::TokKind TGLexer::LexBracket() {
520 if (CurPtr[0] != '{')
521 return tgtok::l_square;
522 ++CurPtr;
523 const char *CodeStart = CurPtr;
524 while (true) {
525 int Char = getNextChar();
526 if (Char == EOF) break;
527
528 if (Char != '}') continue;
529
530 Char = getNextChar();
531 if (Char == EOF) break;
532 if (Char == ']') {
533 CurStrVal.assign(CodeStart, CurPtr-2);
534 return tgtok::CodeFragment;
535 }
536 }
537
538 return ReturnError(CodeStart - 2, "Unterminated code block");
539}
540
541/// LexExclaim - Lex '!' and '![a-zA-Z]+'.
542tgtok::TokKind TGLexer::LexExclaim() {
543 if (!isalpha(*CurPtr))
544 return ReturnError(CurPtr - 1, "Invalid \"!operator\"");
545
546 const char *Start = CurPtr++;
547 while (isalpha(*CurPtr))
548 ++CurPtr;
549
550 // Check to see which operator this is.
552 StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))
553 .Case("eq", tgtok::XEq)
554 .Case("ne", tgtok::XNe)
555 .Case("le", tgtok::XLe)
556 .Case("lt", tgtok::XLt)
557 .Case("ge", tgtok::XGe)
558 .Case("gt", tgtok::XGt)
559 .Case("if", tgtok::XIf)
560 .Case("cond", tgtok::XCond)
561 .Case("isa", tgtok::XIsA)
562 .Case("head", tgtok::XHead)
563 .Case("tail", tgtok::XTail)
564 .Case("size", tgtok::XSize)
565 .Case("con", tgtok::XConcat)
566 .Case("dag", tgtok::XDag)
567 .Case("add", tgtok::XADD)
568 .Case("sub", tgtok::XSUB)
569 .Case("mul", tgtok::XMUL)
570 .Case("div", tgtok::XDIV)
571 .Case("not", tgtok::XNOT)
572 .Case("logtwo", tgtok::XLOG2)
573 .Case("and", tgtok::XAND)
574 .Case("or", tgtok::XOR)
575 .Case("xor", tgtok::XXOR)
576 .Case("shl", tgtok::XSHL)
577 .Case("sra", tgtok::XSRA)
578 .Case("srl", tgtok::XSRL)
579 .Case("cast", tgtok::XCast)
580 .Case("empty", tgtok::XEmpty)
581 .Case("subst", tgtok::XSubst)
582 .Case("foldl", tgtok::XFoldl)
583 .Case("foreach", tgtok::XForEach)
584 .Case("filter", tgtok::XFilter)
585 .Case("listconcat", tgtok::XListConcat)
586 .Case("listsplat", tgtok::XListSplat)
587 .Case("listremove", tgtok::XListRemove)
588 .Case("strconcat", tgtok::XStrConcat)
589 .Case("interleave", tgtok::XInterleave)
590 .Case("substr", tgtok::XSubstr)
591 .Case("find", tgtok::XFind)
592 .Cases("setdagop", "setop", tgtok::XSetDagOp) // !setop is deprecated.
593 .Cases("getdagop", "getop", tgtok::XGetDagOp) // !getop is deprecated.
594 .Case("exists", tgtok::XExists)
595 .Case("tolower", tgtok::XToLower)
596 .Case("toupper", tgtok::XToUpper)
598
599 return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
600}
601
602bool TGLexer::prepExitInclude(bool IncludeStackMustBeEmpty) {
603 // Report an error, if preprocessor control stack for the current
604 // file is not empty.
605 if (!PrepIncludeStack.back()->empty()) {
606 prepReportPreprocessorStackError();
607
608 return false;
609 }
610
611 // Pop the preprocessing controls from the include stack.
612 if (PrepIncludeStack.empty()) {
613 PrintFatalError("Preprocessor include stack is empty");
614 }
615
616 PrepIncludeStack.pop_back();
617
618 if (IncludeStackMustBeEmpty) {
619 if (!PrepIncludeStack.empty())
620 PrintFatalError("Preprocessor include stack is not empty");
621 } else {
622 if (PrepIncludeStack.empty())
623 PrintFatalError("Preprocessor include stack is empty");
624 }
625
626 return true;
627}
628
629tgtok::TokKind TGLexer::prepIsDirective() const {
630 for (const auto &PD : PreprocessorDirs) {
631 int NextChar = *CurPtr;
632 bool Match = true;
633 unsigned I = 0;
634 for (; I < strlen(PD.Word); ++I) {
635 if (NextChar != PD.Word[I]) {
636 Match = false;
637 break;
638 }
639
640 NextChar = peekNextChar(I + 1);
641 }
642
643 // Check for whitespace after the directive. If there is no whitespace,
644 // then we do not recognize it as a preprocessing directive.
645 if (Match) {
646 tgtok::TokKind Kind = PD.Kind;
647
648 // New line and EOF may follow only #else/#endif. It will be reported
649 // as an error for #ifdef/#define after the call to prepLexMacroName().
650 if (NextChar == ' ' || NextChar == '\t' || NextChar == EOF ||
651 NextChar == '\n' ||
652 // It looks like TableGen does not support '\r' as the actual
653 // carriage return, e.g. getNextChar() treats a single '\r'
654 // as '\n'. So we do the same here.
655 NextChar == '\r')
656 return Kind;
657
658 // Allow comments after some directives, e.g.:
659 // #else// OR #else/**/
660 // #endif// OR #endif/**/
661 //
662 // Note that we do allow comments after #ifdef/#define here, e.g.
663 // #ifdef/**/ AND #ifdef//
664 // #define/**/ AND #define//
665 //
666 // These cases will be reported as incorrect after calling
667 // prepLexMacroName(). We could have supported C-style comments
668 // after #ifdef/#define, but this would complicate the code
669 // for little benefit.
670 if (NextChar == '/') {
671 NextChar = peekNextChar(I + 1);
672
673 if (NextChar == '*' || NextChar == '/')
674 return Kind;
675
676 // Pretend that we do not recognize the directive.
677 }
678 }
679 }
680
681 return tgtok::Error;
682}
683
684bool TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
685 TokStart = CurPtr;
686
687 for (const auto &PD : PreprocessorDirs)
688 if (PD.Kind == Kind) {
689 // Advance CurPtr to the end of the preprocessing word.
690 CurPtr += strlen(PD.Word);
691 return true;
692 }
693
694 PrintFatalError("Unsupported preprocessing token in "
695 "prepEatPreprocessorDirective()");
696 return false;
697}
698
699tgtok::TokKind TGLexer::lexPreprocessor(
700 tgtok::TokKind Kind, bool ReturnNextLiveToken) {
701
702 // We must be looking at a preprocessing directive. Eat it!
703 if (!prepEatPreprocessorDirective(Kind))
704 PrintFatalError("lexPreprocessor() called for unknown "
705 "preprocessor directive");
706
707 if (Kind == tgtok::Ifdef || Kind == tgtok::Ifndef) {
708 StringRef MacroName = prepLexMacroName();
709 StringRef IfTokName = Kind == tgtok::Ifdef ? "#ifdef" : "#ifndef";
710 if (MacroName.empty())
711 return ReturnError(TokStart, "Expected macro name after " + IfTokName);
712
713 bool MacroIsDefined = DefinedMacros.count(MacroName) != 0;
714
715 // Canonicalize ifndef to ifdef equivalent
716 if (Kind == tgtok::Ifndef) {
717 MacroIsDefined = !MacroIsDefined;
719 }
720
721 // Regardless of whether we are processing tokens or not,
722 // we put the #ifdef control on stack.
723 PrepIncludeStack.back()->push_back(
724 {Kind, MacroIsDefined, SMLoc::getFromPointer(TokStart)});
725
726 if (!prepSkipDirectiveEnd())
727 return ReturnError(CurPtr, "Only comments are supported after " +
728 IfTokName + " NAME");
729
730 // If we were not processing tokens before this #ifdef,
731 // then just return back to the lines skipping code.
732 if (!ReturnNextLiveToken)
733 return Kind;
734
735 // If we were processing tokens before this #ifdef,
736 // and the macro is defined, then just return the next token.
737 if (MacroIsDefined)
738 return LexToken();
739
740 // We were processing tokens before this #ifdef, and the macro
741 // is not defined, so we have to start skipping the lines.
742 // If the skipping is successful, it will return the token following
743 // either #else or #endif corresponding to this #ifdef.
744 if (prepSkipRegion(ReturnNextLiveToken))
745 return LexToken();
746
747 return tgtok::Error;
748 } else if (Kind == tgtok::Else) {
749 // Check if this #else is correct before calling prepSkipDirectiveEnd(),
750 // which will move CurPtr away from the beginning of #else.
751 if (PrepIncludeStack.back()->empty())
752 return ReturnError(TokStart, "#else without #ifdef or #ifndef");
753
754 PreprocessorControlDesc IfdefEntry = PrepIncludeStack.back()->back();
755
756 if (IfdefEntry.Kind != tgtok::Ifdef) {
757 PrintError(TokStart, "double #else");
758 return ReturnError(IfdefEntry.SrcPos, "Previous #else is here");
759 }
760
761 // Replace the corresponding #ifdef's control with its negation
762 // on the control stack.
763 PrepIncludeStack.back()->pop_back();
764 PrepIncludeStack.back()->push_back(
765 {Kind, !IfdefEntry.IsDefined, SMLoc::getFromPointer(TokStart)});
766
767 if (!prepSkipDirectiveEnd())
768 return ReturnError(CurPtr, "Only comments are supported after #else");
769
770 // If we were processing tokens before this #else,
771 // we have to start skipping lines until the matching #endif.
772 if (ReturnNextLiveToken) {
773 if (prepSkipRegion(ReturnNextLiveToken))
774 return LexToken();
775
776 return tgtok::Error;
777 }
778
779 // Return to the lines skipping code.
780 return Kind;
781 } else if (Kind == tgtok::Endif) {
782 // Check if this #endif is correct before calling prepSkipDirectiveEnd(),
783 // which will move CurPtr away from the beginning of #endif.
784 if (PrepIncludeStack.back()->empty())
785 return ReturnError(TokStart, "#endif without #ifdef");
786
787 auto &IfdefOrElseEntry = PrepIncludeStack.back()->back();
788
789 if (IfdefOrElseEntry.Kind != tgtok::Ifdef &&
790 IfdefOrElseEntry.Kind != tgtok::Else) {
791 PrintFatalError("Invalid preprocessor control on the stack");
792 return tgtok::Error;
793 }
794
795 if (!prepSkipDirectiveEnd())
796 return ReturnError(CurPtr, "Only comments are supported after #endif");
797
798 PrepIncludeStack.back()->pop_back();
799
800 // If we were processing tokens before this #endif, then
801 // we should continue it.
802 if (ReturnNextLiveToken) {
803 return LexToken();
804 }
805
806 // Return to the lines skipping code.
807 return Kind;
808 } else if (Kind == tgtok::Define) {
809 StringRef MacroName = prepLexMacroName();
810 if (MacroName.empty())
811 return ReturnError(TokStart, "Expected macro name after #define");
812
813 if (!DefinedMacros.insert(MacroName).second)
815 "Duplicate definition of macro: " + Twine(MacroName));
816
817 if (!prepSkipDirectiveEnd())
818 return ReturnError(CurPtr,
819 "Only comments are supported after #define NAME");
820
821 if (!ReturnNextLiveToken) {
822 PrintFatalError("#define must be ignored during the lines skipping");
823 return tgtok::Error;
824 }
825
826 return LexToken();
827 }
828
829 PrintFatalError("Preprocessing directive is not supported");
830 return tgtok::Error;
831}
832
833bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
834 if (!MustNeverBeFalse)
835 PrintFatalError("Invalid recursion.");
836
837 do {
838 // Skip all symbols to the line end.
839 prepSkipToLineEnd();
840
841 // Find the first non-whitespace symbol in the next line(s).
842 if (!prepSkipLineBegin())
843 return false;
844
845 // If the first non-blank/comment symbol on the line is '#',
846 // it may be a start of preprocessing directive.
847 //
848 // If it is not '#' just go to the next line.
849 if (*CurPtr == '#')
850 ++CurPtr;
851 else
852 continue;
853
854 tgtok::TokKind Kind = prepIsDirective();
855
856 // If we did not find a preprocessing directive or it is #define,
857 // then just skip to the next line. We do not have to do anything
858 // for #define in the line-skipping mode.
859 if (Kind == tgtok::Error || Kind == tgtok::Define)
860 continue;
861
862 tgtok::TokKind ProcessedKind = lexPreprocessor(Kind, false);
863
864 // If lexPreprocessor() encountered an error during lexing this
865 // preprocessor idiom, then return false to the calling lexPreprocessor().
866 // This will force tgtok::Error to be returned to the tokens processing.
867 if (ProcessedKind == tgtok::Error)
868 return false;
869
870 if (Kind != ProcessedKind)
871 PrintFatalError("prepIsDirective() and lexPreprocessor() "
872 "returned different token kinds");
873
874 // If this preprocessing directive enables tokens processing,
875 // then return to the lexPreprocessor() and get to the next token.
876 // We can move from line-skipping mode to processing tokens only
877 // due to #else or #endif.
878 if (prepIsProcessingEnabled()) {
879 if (Kind != tgtok::Else && Kind != tgtok::Endif) {
880 PrintFatalError("Tokens processing was enabled by an unexpected "
881 "preprocessing directive");
882 return false;
883 }
884
885 return true;
886 }
887 } while (CurPtr != CurBuf.end());
888
889 // We have reached the end of the file, but never left the lines-skipping
890 // mode. This means there is no matching #endif.
891 prepReportPreprocessorStackError();
892 return false;
893}
894
895StringRef TGLexer::prepLexMacroName() {
896 // Skip whitespaces between the preprocessing directive and the macro name.
897 while (*CurPtr == ' ' || *CurPtr == '\t')
898 ++CurPtr;
899
900 TokStart = CurPtr;
901 // Macro names start with [a-zA-Z_].
902 if (*CurPtr != '_' && !isalpha(*CurPtr))
903 return "";
904
905 // Match the rest of the identifier regex: [0-9a-zA-Z_]*
906 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
907 ++CurPtr;
908
909 return StringRef(TokStart, CurPtr - TokStart);
910}
911
912bool TGLexer::prepSkipLineBegin() {
913 while (CurPtr != CurBuf.end()) {
914 switch (*CurPtr) {
915 case ' ':
916 case '\t':
917 case '\n':
918 case '\r':
919 break;
920
921 case '/': {
922 int NextChar = peekNextChar(1);
923 if (NextChar == '*') {
924 // Skip C-style comment.
925 // Note that we do not care about skipping the C++-style comments.
926 // If the line contains "//", it may not contain any processable
927 // preprocessing directive. Just return CurPtr pointing to
928 // the first '/' in this case. We also do not care about
929 // incorrect symbols after the first '/' - we are in lines-skipping
930 // mode, so incorrect code is allowed to some extent.
931
932 // Set TokStart to the beginning of the comment to enable proper
933 // diagnostic printing in case of error in SkipCComment().
934 TokStart = CurPtr;
935
936 // CurPtr must point to '*' before call to SkipCComment().
937 ++CurPtr;
938 if (SkipCComment())
939 return false;
940 } else {
941 // CurPtr points to the non-whitespace '/'.
942 return true;
943 }
944
945 // We must not increment CurPtr after the comment was lexed.
946 continue;
947 }
948
949 default:
950 return true;
951 }
952
953 ++CurPtr;
954 }
955
956 // We have reached the end of the file. Return to the lines skipping
957 // code, and allow it to handle the EOF as needed.
958 return true;
959}
960
961bool TGLexer::prepSkipDirectiveEnd() {
962 while (CurPtr != CurBuf.end()) {
963 switch (*CurPtr) {
964 case ' ':
965 case '\t':
966 break;
967
968 case '\n':
969 case '\r':
970 return true;
971
972 case '/': {
973 int NextChar = peekNextChar(1);
974 if (NextChar == '/') {
975 // Skip C++-style comment.
976 // We may just return true now, but let's skip to the line/buffer end
977 // to simplify the method specification.
978 ++CurPtr;
979 SkipBCPLComment();
980 } else if (NextChar == '*') {
981 // When we are skipping C-style comment at the end of a preprocessing
982 // directive, we can skip several lines. If any meaningful TD token
983 // follows the end of the C-style comment on the same line, it will
984 // be considered as an invalid usage of TD token.
985 // For example, we want to forbid usages like this one:
986 // #define MACRO class Class {}
987 // But with C-style comments we also disallow the following:
988 // #define MACRO /* This macro is used
989 // to ... */ class Class {}
990 // One can argue that this should be allowed, but it does not seem
991 // to be worth of the complication. Moreover, this matches
992 // the C preprocessor behavior.
993
994 // Set TokStart to the beginning of the comment to enable proper
995 // diagnostic printer in case of error in SkipCComment().
996 TokStart = CurPtr;
997 ++CurPtr;
998 if (SkipCComment())
999 return false;
1000 } else {
1001 TokStart = CurPtr;
1002 PrintError(CurPtr, "Unexpected character");
1003 return false;
1004 }
1005
1006 // We must not increment CurPtr after the comment was lexed.
1007 continue;
1008 }
1009
1010 default:
1011 // Do not allow any non-whitespaces after the directive.
1012 TokStart = CurPtr;
1013 return false;
1014 }
1015
1016 ++CurPtr;
1017 }
1018
1019 return true;
1020}
1021
1022void TGLexer::prepSkipToLineEnd() {
1023 while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end())
1024 ++CurPtr;
1025}
1026
1027bool TGLexer::prepIsProcessingEnabled() {
1028 for (const PreprocessorControlDesc &I :
1029 llvm::reverse(*PrepIncludeStack.back()))
1030 if (!I.IsDefined)
1031 return false;
1032
1033 return true;
1034}
1035
1036void TGLexer::prepReportPreprocessorStackError() {
1037 if (PrepIncludeStack.back()->empty())
1038 PrintFatalError("prepReportPreprocessorStackError() called with "
1039 "empty control stack");
1040
1041 auto &PrepControl = PrepIncludeStack.back()->back();
1042 PrintError(CurBuf.end(), "Reached EOF without matching #endif");
1043 PrintError(PrepControl.SrcPos, "The latest preprocessor control is here");
1044
1045 TokStart = CurPtr;
1046}
#define I(x, y, z)
Definition: MD5.cpp:58
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
StringRef getBuffer() const
Definition: MemoryBuffer.h:70
Represents a location in source code.
Definition: SMLoc.h:23
static SMLoc getFromPointer(const char *Ptr)
Definition: SMLoc.h:36
const char * getPointer() const
Definition: SMLoc.h:34
Represents a range in source code.
Definition: SMLoc.h:48
This owns the files read by a parser, handles include stacks, and handles diagnostic wrangling.
Definition: SourceMgr.h:31
unsigned getMainFileID() const
Definition: SourceMgr.h:132
const MemoryBuffer * getMemoryBuffer(unsigned i) const
Definition: SourceMgr.h:125
SMLoc getParentIncludeLoc(unsigned i) const
Definition: SourceMgr.h:137
unsigned FindBufferContainingLoc(SMLoc Loc) const
Return the ID of the buffer containing the specified location.
Definition: SourceMgr.cpp:73
unsigned AddIncludeFile(const std::string &Filename, SMLoc IncludeLoc, std::string &IncludedFile)
Search for a file with the specified name in the current directory or in one of the IncludeDirs.
Definition: SourceMgr.cpp:41
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:256
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
iterator begin() const
Definition: StringRef.h:111
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:375
iterator end() const
Definition: StringRef.h:113
static constexpr size_t npos
Definition: StringRef.h:52
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
std::pair< typename Base::iterator, bool > insert(StringRef key)
Definition: StringSet.h:34
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:44
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:69
R Default(T Value)
Definition: StringSwitch.h:182
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Definition: StringSwitch.h:90
SMRange getLocRange() const
Definition: TGLexer.cpp:66
tgtok::TokKind Lex()
Definition: TGLexer.h:107
SMLoc getLoc() const
Definition: TGLexer.cpp:62
TGLexer(SourceMgr &SrcMgr, ArrayRef< std::string > Macros)
Definition: TGLexer.cpp:47
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
support::ulittle32_t Word
Definition: IRSymtab.h:52
@ r_square
Definition: TGLexer.h:38
@ XListSplat
Definition: TGLexer.h:56
@ l_square
Definition: TGLexer.h:38
@ CodeFragment
Definition: TGLexer.h:72
@ XInterleave
Definition: TGLexer.h:56
@ MultiClass
Definition: TGLexer.h:51
@ BinaryIntVal
Definition: TGLexer.h:69
@ XToLower
Definition: TGLexer.h:59
@ XGetDagOp
Definition: TGLexer.h:58
@ XToUpper
Definition: TGLexer.h:59
@ XForEach
Definition: TGLexer.h:57
@ XListConcat
Definition: TGLexer.h:56
@ XSetDagOp
Definition: TGLexer.h:58
@ XStrConcat
Definition: TGLexer.h:56
@ FalseVal
Definition: TGLexer.h:62
@ dotdotdot
Definition: TGLexer.h:46
@ question
Definition: TGLexer.h:44
@ XListRemove
Definition: TGLexer.h:59
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void PrintFatalError(const Twine &Msg)
Definition: Error.cpp:125
void PrintError(const Twine &Msg)
Definition: Error.cpp:101
SourceMgr SrcMgr
Definition: Error.cpp:24
void PrintWarning(const Twine &Msg)
Definition: Error.cpp:89
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:511