LLVM 18.0.0git
TGLexer.h
Go to the documentation of this file.
1//===- TGLexer.h - Lexer for TableGen Files ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This class represents the Lexer for tablegen files.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TABLEGEN_TGLEXER_H
14#define LLVM_LIB_TABLEGEN_TGLEXER_H
15
16#include "llvm/ADT/StringRef.h"
17#include "llvm/ADT/StringSet.h"
19#include "llvm/Support/SMLoc.h"
20#include <cassert>
21#include <memory>
22#include <set>
23#include <string>
24#include <vector>
25
26namespace llvm {
27template <typename T> class ArrayRef;
28class SourceMgr;
29class Twine;
30
31namespace tgtok {
32enum TokKind {
33 // Markers
36
37 // Tokens with no info.
38 minus, // -
39 plus, // +
42 l_brace, // {
43 r_brace, // }
44 l_paren, // (
45 r_paren, // )
46 less, // <
47 greater, // >
48 colon, // :
49 semi, // ;
50 comma, // ,
51 dot, // .
52 equal, // =
54 paste, // #
55 dotdotdot, // ...
56
57 // Boolean literals.
60
61 // Integer value.
63
64 // Binary constant. Note that these are sized according to the number of
65 // bits given.
67
68 // Preprocessing tokens for internal usage by the lexer.
69 // They are never returned as a result of Lex().
75
76 // Reserved keywords. ('ElseKW' is named to distinguish it from the
77 // existing 'Else' that means the preprocessor #else.)
92
93 // Object start tokens.
106
107 // Bang operators.
159
160 // String valued tokens.
167};
168
169/// isBangOperator - Return true if this is a bang operator.
170static inline bool isBangOperator(tgtok::TokKind Kind) {
171 return tgtok::BANG_OPERATOR_FIRST <= Kind && Kind <= BANG_OPERATOR_LAST;
172}
173
174/// isObjectStart - Return true if this is a valid first token for a statement.
175static inline bool isObjectStart(tgtok::TokKind Kind) {
176 return tgtok::OBJECT_START_FIRST <= Kind && Kind <= OBJECT_START_LAST;
177}
178
179/// isStringValue - Return true if this is a string value.
180static inline bool isStringValue(tgtok::TokKind Kind) {
181 return tgtok::STRING_VALUE_FIRST <= Kind && Kind <= STRING_VALUE_LAST;
182}
183} // namespace tgtok
184
185/// TGLexer - TableGen Lexer class.
186class TGLexer {
187 SourceMgr &SrcMgr;
188
189 const char *CurPtr = nullptr;
190 StringRef CurBuf;
191
192 // Information about the current token.
193 const char *TokStart = nullptr;
195 std::string CurStrVal; // This is valid for Id, StrVal, VarName, CodeFragment
196 int64_t CurIntVal = 0; // This is valid for IntVal.
197
198 /// CurBuffer - This is the current buffer index we're lexing from as managed
199 /// by the SourceMgr object.
200 unsigned CurBuffer = 0;
201
202public:
203 typedef std::set<std::string> DependenciesSetTy;
204
205private:
206 /// Dependencies - This is the list of all included files.
207 DependenciesSetTy Dependencies;
208
209public:
210 TGLexer(SourceMgr &SrcMgr, ArrayRef<std::string> Macros);
211
213 return CurCode = LexToken(CurPtr == CurBuf.begin());
214 }
215
217 return Dependencies;
218 }
219
220 tgtok::TokKind getCode() const { return CurCode; }
221
222 const std::string &getCurStrVal() const {
223 assert(tgtok::isStringValue(CurCode) &&
224 "This token doesn't have a string value");
225 return CurStrVal;
226 }
227 int64_t getCurIntVal() const {
228 assert(CurCode == tgtok::IntVal && "This token isn't an integer");
229 return CurIntVal;
230 }
231 std::pair<int64_t, unsigned> getCurBinaryIntVal() const {
232 assert(CurCode == tgtok::BinaryIntVal &&
233 "This token isn't a binary integer");
234 return std::make_pair(CurIntVal, (CurPtr - TokStart)-2);
235 }
236
237 SMLoc getLoc() const;
238 SMRange getLocRange() const;
239
240private:
241 /// LexToken - Read the next token and return its code.
242 tgtok::TokKind LexToken(bool FileOrLineStart = false);
243
244 tgtok::TokKind ReturnError(SMLoc Loc, const Twine &Msg);
245 tgtok::TokKind ReturnError(const char *Loc, const Twine &Msg);
246
247 int getNextChar();
248 int peekNextChar(int Index) const;
249 void SkipBCPLComment();
250 bool SkipCComment();
251 tgtok::TokKind LexIdentifier();
252 bool LexInclude();
253 tgtok::TokKind LexString();
254 tgtok::TokKind LexVarName();
255 tgtok::TokKind LexNumber();
256 tgtok::TokKind LexBracket();
257 tgtok::TokKind LexExclaim();
258
259 // Process EOF encountered in LexToken().
260 // If EOF is met in an include file, then the method will update
261 // CurPtr, CurBuf and preprocessing include stack, and return true.
262 // If EOF is met in the top-level file, then the method will
263 // update and check the preprocessing include stack, and return false.
264 bool processEOF();
265
266 // *** Structures and methods for preprocessing support ***
267
268 // A set of macro names that are defined either via command line or
269 // by using:
270 // #define NAME
271 StringSet<> DefinedMacros;
272
273 // Each of #ifdef and #else directives has a descriptor associated
274 // with it.
275 //
276 // An ordered list of preprocessing controls defined by #ifdef/#else
277 // directives that are in effect currently is called preprocessing
278 // control stack. It is represented as a vector of PreprocessorControlDesc's.
279 //
280 // The control stack is updated according to the following rules:
281 //
282 // For each #ifdef we add an element to the control stack.
283 // For each #else we replace the top element with a descriptor
284 // with an inverted IsDefined value.
285 // For each #endif we pop the top element from the control stack.
286 //
287 // When CurPtr reaches the current buffer's end, the control stack
288 // must be empty, i.e. #ifdef and the corresponding #endif
289 // must be located in the same file.
290 struct PreprocessorControlDesc {
291 // Either tgtok::Ifdef or tgtok::Else.
292 tgtok::TokKind Kind;
293
294 // True, if the condition for this directive is true, false - otherwise.
295 // Examples:
296 // #ifdef NAME : true, if NAME is defined, false - otherwise.
297 // ...
298 // #else : false, if NAME is defined, true - otherwise.
299 bool IsDefined;
300
301 // Pointer into CurBuf to the beginning of the preprocessing directive
302 // word, e.g.:
303 // #ifdef NAME
304 // ^ - SrcPos
305 SMLoc SrcPos;
306 };
307
308 // We want to disallow code like this:
309 // file1.td:
310 // #define NAME
311 // #ifdef NAME
312 // include "file2.td"
313 // EOF
314 // file2.td:
315 // #endif
316 // EOF
317 //
318 // To do this, we clear the preprocessing control stack on entry
319 // to each of the included file. PrepIncludeStack is used to store
320 // preprocessing control stacks for the current file and all its
321 // parent files. The back() element is the preprocessing control
322 // stack for the current file.
323 std::vector<std::unique_ptr<std::vector<PreprocessorControlDesc>>>
324 PrepIncludeStack;
325
326 // Validate that the current preprocessing control stack is empty,
327 // since we are about to exit a file, and pop the include stack.
328 //
329 // If IncludeStackMustBeEmpty is true, the include stack must be empty
330 // after the popping, otherwise, the include stack must not be empty
331 // after the popping. Basically, the include stack must be empty
332 // only if we exit the "top-level" file (i.e. finish lexing).
333 //
334 // The method returns false, if the current preprocessing control stack
335 // is not empty (e.g. there is an unterminated #ifdef/#else),
336 // true - otherwise.
337 bool prepExitInclude(bool IncludeStackMustBeEmpty);
338
339 // Look ahead for a preprocessing directive starting from CurPtr. The caller
340 // must only call this method, if *(CurPtr - 1) is '#'. If the method matches
341 // a preprocessing directive word followed by a whitespace, then it returns
342 // one of the internal token kinds, i.e. Ifdef, Else, Endif, Define.
343 //
344 // CurPtr is not adjusted by this method.
345 tgtok::TokKind prepIsDirective() const;
346
347 // Given a preprocessing token kind, adjusts CurPtr to the end
348 // of the preprocessing directive word. Returns true, unless
349 // an unsupported token kind is passed in.
350 //
351 // We use look-ahead prepIsDirective() and prepEatPreprocessorDirective()
352 // to avoid adjusting CurPtr before we are sure that '#' is followed
353 // by a preprocessing directive. If it is not, then we fall back to
354 // tgtok::paste interpretation of '#'.
355 bool prepEatPreprocessorDirective(tgtok::TokKind Kind);
356
357 // The main "exit" point from the token parsing to preprocessor.
358 //
359 // The method is called for CurPtr, when prepIsDirective() returns
360 // true. The first parameter matches the result of prepIsDirective(),
361 // denoting the actual preprocessor directive to be processed.
362 //
363 // If the preprocessing directive disables the tokens processing, e.g.:
364 // #ifdef NAME // NAME is undefined
365 // then lexPreprocessor() enters the lines-skipping mode.
366 // In this mode, it does not parse any tokens, because the code under
367 // the #ifdef may not even be a correct tablegen code. The preprocessor
368 // looks for lines containing other preprocessing directives, which
369 // may be prepended with whitespaces and C-style comments. If the line
370 // does not contain a preprocessing directive, it is skipped completely.
371 // Otherwise, the preprocessing directive is processed by recursively
372 // calling lexPreprocessor(). The processing of the encountered
373 // preprocessing directives includes updating preprocessing control stack
374 // and adding new macros into DefinedMacros set.
375 //
376 // The second parameter controls whether lexPreprocessor() is called from
377 // LexToken() (true) or recursively from lexPreprocessor() (false).
378 //
379 // If ReturnNextLiveToken is true, the method returns the next
380 // LEX token following the current directive or following the end
381 // of the disabled preprocessing region corresponding to this directive.
382 // If ReturnNextLiveToken is false, the method returns the first parameter,
383 // unless there were errors encountered in the disabled preprocessing
384 // region - in this case, it returns tgtok::Error.
385 tgtok::TokKind lexPreprocessor(tgtok::TokKind Kind,
386 bool ReturnNextLiveToken = true);
387
388 // Worker method for lexPreprocessor() to skip lines after some
389 // preprocessing directive up to the buffer end or to the directive
390 // that re-enables token processing. The method returns true
391 // upon processing the next directive that re-enables tokens
392 // processing. False is returned if an error was encountered.
393 //
394 // Note that prepSkipRegion() calls lexPreprocessor() to process
395 // encountered preprocessing directives. In this case, the second
396 // parameter to lexPreprocessor() is set to false. Being passed
397 // false ReturnNextLiveToken, lexPreprocessor() must never call
398 // prepSkipRegion(). We assert this by passing ReturnNextLiveToken
399 // to prepSkipRegion() and checking that it is never set to false.
400 bool prepSkipRegion(bool MustNeverBeFalse);
401
402 // Lex name of the macro after either #ifdef or #define. We could have used
403 // LexIdentifier(), but it has special handling of "include" word, which
404 // could result in awkward diagnostic errors. Consider:
405 // ----
406 // #ifdef include
407 // class ...
408 // ----
409 // LexIdentifier() will engage LexInclude(), which will complain about
410 // missing file with name "class". Instead, prepLexMacroName() will treat
411 // "include" as a normal macro name.
412 //
413 // On entry, CurPtr points to the end of a preprocessing directive word.
414 // The method allows for whitespaces between the preprocessing directive
415 // and the macro name. The allowed whitespaces are ' ' and '\t'.
416 //
417 // If the first non-whitespace symbol after the preprocessing directive
418 // is a valid start symbol for an identifier (i.e. [a-zA-Z_]), then
419 // the method updates TokStart to the position of the first non-whitespace
420 // symbol, sets CurPtr to the position of the macro name's last symbol,
421 // and returns a string reference to the macro name. Otherwise,
422 // TokStart is set to the first non-whitespace symbol after the preprocessing
423 // directive, and the method returns an empty string reference.
424 //
425 // In all cases, TokStart may be used to point to the word following
426 // the preprocessing directive.
427 StringRef prepLexMacroName();
428
429 // Skip any whitespaces starting from CurPtr. The method is used
430 // only in the lines-skipping mode to find the first non-whitespace
431 // symbol after or at CurPtr. Allowed whitespaces are ' ', '\t', '\n'
432 // and '\r'. The method skips C-style comments as well, because
433 // it is used to find the beginning of the preprocessing directive.
434 // If we do not handle C-style comments the following code would
435 // result in incorrect detection of a preprocessing directive:
436 // /*
437 // #ifdef NAME
438 // */
439 // As long as we skip C-style comments, the following code is correctly
440 // recognized as a preprocessing directive:
441 // /* first line comment
442 // second line comment */ #ifdef NAME
443 //
444 // The method returns true upon reaching the first non-whitespace symbol
445 // or EOF, CurPtr is set to point to this symbol. The method returns false,
446 // if an error occurred during skipping of a C-style comment.
447 bool prepSkipLineBegin();
448
449 // Skip any whitespaces or comments after a preprocessing directive.
450 // The method returns true upon reaching either end of the line
451 // or end of the file. If there is a multiline C-style comment
452 // after the preprocessing directive, the method skips
453 // the comment, so the final CurPtr may point to one of the next lines.
454 // The method returns false, if an error occurred during skipping
455 // C- or C++-style comment, or a non-whitespace symbol appears
456 // after the preprocessing directive.
457 //
458 // The method maybe called both during lines-skipping and tokens
459 // processing. It actually verifies that only whitespaces or/and
460 // comments follow a preprocessing directive.
461 //
462 // After the execution of this mehod, CurPtr points either to new line
463 // symbol, buffer end or non-whitespace symbol following the preprocesing
464 // directive.
465 bool prepSkipDirectiveEnd();
466
467 // Skip all symbols to the end of the line/file.
468 // The method adjusts CurPtr, so that it points to either new line
469 // symbol in the current line or the buffer end.
470 void prepSkipToLineEnd();
471
472 // Return true, if the current preprocessor control stack is such that
473 // we should allow lexer to process the next token, false - otherwise.
474 //
475 // In particular, the method returns true, if all the #ifdef/#else
476 // controls on the stack have their IsDefined member set to true.
477 bool prepIsProcessingEnabled();
478
479 // Report an error, if we reach EOF with non-empty preprocessing control
480 // stack. This means there is no matching #endif for the previous
481 // #ifdef/#else.
482 void prepReportPreprocessorStackError();
483};
484
485} // end namespace llvm
486
487#endif
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringSet - A set-like wrapper for the StringMap.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Represents a location in source code.
Definition: SMLoc.h:23
Represents a range in source code.
Definition: SMLoc.h:48
This owns the files read by a parser, handles include stacks, and handles diagnostic wrangling.
Definition: SourceMgr.h:31
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
iterator begin() const
Definition: StringRef.h:111
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
TGLexer - TableGen Lexer class.
Definition: TGLexer.h:186
SMRange getLocRange() const
Definition: TGLexer.cpp:66
tgtok::TokKind Lex()
Definition: TGLexer.h:212
int64_t getCurIntVal() const
Definition: TGLexer.h:227
std::pair< int64_t, unsigned > getCurBinaryIntVal() const
Definition: TGLexer.h:231
const std::string & getCurStrVal() const
Definition: TGLexer.h:222
tgtok::TokKind getCode() const
Definition: TGLexer.h:220
SMLoc getLoc() const
Definition: TGLexer.cpp:62
std::set< std::string > DependenciesSetTy
Definition: TGLexer.h:203
const DependenciesSetTy & getDependencies() const
Definition: TGLexer.h:216
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static bool isBangOperator(tgtok::TokKind Kind)
isBangOperator - Return true if this is a bang operator.
Definition: TGLexer.h:170
@ r_square
Definition: TGLexer.h:41
@ XListSplat
Definition: TGLexer.h:123
@ XSetDagArg
Definition: TGLexer.h:156
@ BANG_OPERATOR_FIRST
Definition: TGLexer.h:108
@ XGetDagName
Definition: TGLexer.h:155
@ STRING_VALUE_FIRST
Definition: TGLexer.h:161
@ l_square
Definition: TGLexer.h:40
@ CodeFragment
Definition: TGLexer.h:165
@ XInterleave
Definition: TGLexer.h:125
@ MultiClass
Definition: TGLexer.h:104
@ BinaryIntVal
Definition: TGLexer.h:66
@ XSetDagName
Definition: TGLexer.h:157
@ BANG_OPERATOR_LAST
Definition: TGLexer.h:158
@ OBJECT_START_LAST
Definition: TGLexer.h:105
@ STRING_VALUE_LAST
Definition: TGLexer.h:166
@ XGetDagArg
Definition: TGLexer.h:154
@ XListConcat
Definition: TGLexer.h:122
@ OBJECT_START_FIRST
Definition: TGLexer.h:94
@ XStrConcat
Definition: TGLexer.h:124
@ FalseVal
Definition: TGLexer.h:59
@ dotdotdot
Definition: TGLexer.h:55
@ question
Definition: TGLexer.h:53
@ XListRemove
Definition: TGLexer.h:150
static bool isObjectStart(tgtok::TokKind Kind)
isObjectStart - Return true if this is a valid first token for a statement.
Definition: TGLexer.h:175
static bool isStringValue(tgtok::TokKind Kind)
isStringValue - Return true if this is a string value.
Definition: TGLexer.h:180
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
ArrayRef(const T &OneElt) -> ArrayRef< T >