clang  3.9.0
PPLexerChange.cpp
Go to the documentation of this file.
1 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements pieces of the Preprocessor interface that manage the
11 // current lexer stack.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Lex/Preprocessor.h"
18 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/MacroInfo.h"
21 #include "clang/Lex/PTHManager.h"
22 #include "llvm/ADT/StringSwitch.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/MemoryBuffer.h"
25 #include "llvm/Support/Path.h"
26 using namespace clang;
27 
29 
30 //===----------------------------------------------------------------------===//
31 // Miscellaneous Methods.
32 //===----------------------------------------------------------------------===//
33 
34 /// isInPrimaryFile - Return true if we're in the top-level file, not in a
35 /// \#include. This looks through macro expansions and active _Pragma lexers.
37  if (IsFileLexer())
38  return IncludeMacroStack.empty();
39 
40  // If there are any stacked lexers, we're in a #include.
41  assert(IsFileLexer(IncludeMacroStack[0]) &&
42  "Top level include stack isn't our primary lexer?");
43  for (unsigned i = 1, e = IncludeMacroStack.size(); i != e; ++i)
44  if (IsFileLexer(IncludeMacroStack[i]))
45  return false;
46  return true;
47 }
48 
49 /// getCurrentLexer - Return the current file lexer being lexed from. Note
50 /// that this ignores any potentially active macro expansions and _Pragma
51 /// expansions going on at the time.
53  if (IsFileLexer())
54  return CurPPLexer;
55 
56  // Look for a stacked lexer.
57  for (unsigned i = IncludeMacroStack.size(); i != 0; --i) {
58  const IncludeStackInfo& ISI = IncludeMacroStack[i-1];
59  if (IsFileLexer(ISI))
60  return ISI.ThePPLexer;
61  }
62  return nullptr;
63 }
64 
65 
66 //===----------------------------------------------------------------------===//
67 // Methods for Entering and Callbacks for leaving various contexts
68 //===----------------------------------------------------------------------===//
69 
70 /// EnterSourceFile - Add a source file to the top of the include stack and
71 /// start lexing tokens from it instead of the current buffer.
73  SourceLocation Loc) {
74  assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
75  ++NumEnteredSourceFiles;
76 
77  if (MaxIncludeStackDepth < IncludeMacroStack.size())
78  MaxIncludeStackDepth = IncludeMacroStack.size();
79 
80  if (PTH) {
81  if (PTHLexer *PL = PTH->CreateLexer(FID)) {
82  EnterSourceFileWithPTH(PL, CurDir);
83  return false;
84  }
85  }
86 
87  // Get the MemoryBuffer for this FID, if it fails, we fail.
88  bool Invalid = false;
89  const llvm::MemoryBuffer *InputFile =
90  getSourceManager().getBuffer(FID, Loc, &Invalid);
91  if (Invalid) {
92  SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
93  Diag(Loc, diag::err_pp_error_opening_file)
94  << std::string(SourceMgr.getBufferName(FileStart)) << "";
95  return true;
96  }
97 
99  SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
100  CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
101  CodeCompletionLoc =
102  CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
103  }
104 
105  EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
106  return false;
107 }
108 
109 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
110 /// and start lexing tokens from it instead of the current buffer.
111 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
112  const DirectoryLookup *CurDir) {
113 
114  // Add the current lexer to the include stack.
115  if (CurPPLexer || CurTokenLexer)
116  PushIncludeMacroStack();
117 
118  CurLexer.reset(TheLexer);
119  CurPPLexer = TheLexer;
120  CurDirLookup = CurDir;
121  CurSubmodule = nullptr;
122  if (CurLexerKind != CLK_LexAfterModuleImport)
123  CurLexerKind = CLK_Lexer;
124 
125  // Notify the client, if desired, that we are in a new source file.
126  if (Callbacks && !CurLexer->Is_PragmaLexer) {
127  SrcMgr::CharacteristicKind FileType =
128  SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
129 
130  Callbacks->FileChanged(CurLexer->getFileLoc(),
131  PPCallbacks::EnterFile, FileType);
132  }
133 }
134 
135 /// EnterSourceFileWithPTH - Add a source file to the top of the include stack
136 /// and start getting tokens from it using the PTH cache.
137 void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
138  const DirectoryLookup *CurDir) {
139 
140  if (CurPPLexer || CurTokenLexer)
141  PushIncludeMacroStack();
142 
143  CurDirLookup = CurDir;
144  CurPTHLexer.reset(PL);
145  CurPPLexer = CurPTHLexer.get();
146  CurSubmodule = nullptr;
147  if (CurLexerKind != CLK_LexAfterModuleImport)
148  CurLexerKind = CLK_PTHLexer;
149 
150  // Notify the client, if desired, that we are in a new source file.
151  if (Callbacks) {
152  FileID FID = CurPPLexer->getFileID();
153  SourceLocation EnterLoc = SourceMgr.getLocForStartOfFile(FID);
154  SrcMgr::CharacteristicKind FileType =
155  SourceMgr.getFileCharacteristic(EnterLoc);
156  Callbacks->FileChanged(EnterLoc, PPCallbacks::EnterFile, FileType);
157  }
158 }
159 
160 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
161 /// tokens from it instead of the current buffer.
163  MacroInfo *Macro, MacroArgs *Args) {
164  std::unique_ptr<TokenLexer> TokLexer;
165  if (NumCachedTokenLexers == 0) {
166  TokLexer = llvm::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
167  } else {
168  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
169  TokLexer->Init(Tok, ILEnd, Macro, Args);
170  }
171 
172  PushIncludeMacroStack();
173  CurDirLookup = nullptr;
174  CurTokenLexer = std::move(TokLexer);
175  if (CurLexerKind != CLK_LexAfterModuleImport)
176  CurLexerKind = CLK_TokenLexer;
177 }
178 
179 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
180 /// which will cause the lexer to start returning the specified tokens.
181 ///
182 /// If DisableMacroExpansion is true, tokens lexed from the token stream will
183 /// not be subject to further macro expansion. Otherwise, these tokens will
184 /// be re-macro-expanded when/if expansion is enabled.
185 ///
186 /// If OwnsTokens is false, this method assumes that the specified stream of
187 /// tokens has a permanent owner somewhere, so they do not need to be copied.
188 /// If it is true, it assumes the array of tokens is allocated with new[] and
189 /// must be freed.
190 ///
191 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
192  bool DisableMacroExpansion,
193  bool OwnsTokens) {
194  if (CurLexerKind == CLK_CachingLexer) {
195  if (CachedLexPos < CachedTokens.size()) {
196  // We're entering tokens into the middle of our cached token stream. We
197  // can't represent that, so just insert the tokens into the buffer.
198  CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
199  Toks, Toks + NumToks);
200  if (OwnsTokens)
201  delete [] Toks;
202  return;
203  }
204 
205  // New tokens are at the end of the cached token sequnece; insert the
206  // token stream underneath the caching lexer.
207  ExitCachingLexMode();
208  EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
209  EnterCachingLexMode();
210  return;
211  }
212 
213  // Create a macro expander to expand from the specified token stream.
214  std::unique_ptr<TokenLexer> TokLexer;
215  if (NumCachedTokenLexers == 0) {
216  TokLexer = llvm::make_unique<TokenLexer>(
217  Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
218  } else {
219  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
220  TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
221  }
222 
223  // Save our current state.
224  PushIncludeMacroStack();
225  CurDirLookup = nullptr;
226  CurTokenLexer = std::move(TokLexer);
227  if (CurLexerKind != CLK_LexAfterModuleImport)
228  CurLexerKind = CLK_TokenLexer;
229 }
230 
231 /// \brief Compute the relative path that names the given file relative to
232 /// the given directory.
233 static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
234  const FileEntry *File,
235  SmallString<128> &Result) {
236  Result.clear();
237 
238  StringRef FilePath = File->getDir()->getName();
239  StringRef Path = FilePath;
240  while (!Path.empty()) {
241  if (const DirectoryEntry *CurDir = FM.getDirectory(Path)) {
242  if (CurDir == Dir) {
243  Result = FilePath.substr(Path.size());
244  llvm::sys::path::append(Result,
245  llvm::sys::path::filename(File->getName()));
246  return;
247  }
248  }
249 
250  Path = llvm::sys::path::parent_path(Path);
251  }
252 
253  Result = File->getName();
254 }
255 
256 void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
257  if (CurTokenLexer) {
258  CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
259  return;
260  }
261  if (CurLexer) {
262  CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
263  return;
264  }
265  // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
266  // but it might if they're empty?
267 }
268 
269 /// \brief Determine the location to use as the end of the buffer for a lexer.
270 ///
271 /// If the file ends with a newline, form the EOF token on the newline itself,
272 /// rather than "on the line following it", which doesn't exist. This makes
273 /// diagnostics relating to the end of file include the last file that the user
274 /// actually typed, which is goodness.
275 const char *Preprocessor::getCurLexerEndPos() {
276  const char *EndPos = CurLexer->BufferEnd;
277  if (EndPos != CurLexer->BufferStart &&
278  (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
279  --EndPos;
280 
281  // Handle \n\r and \r\n:
282  if (EndPos != CurLexer->BufferStart &&
283  (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
284  EndPos[-1] != EndPos[0])
285  --EndPos;
286  }
287 
288  return EndPos;
289 }
290 
291 
292 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
293 /// the current file. This either returns the EOF token or pops a level off
294 /// the include stack and keeps going.
295 bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
296  assert(!CurTokenLexer &&
297  "Ending a file when currently in a macro!");
298 
299  // See if this file had a controlling macro.
300  if (CurPPLexer) { // Not ending a macro, ignore it.
301  if (const IdentifierInfo *ControllingMacro =
302  CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
303  // Okay, this has a controlling macro, remember in HeaderFileInfo.
304  if (const FileEntry *FE = CurPPLexer->getFileEntry()) {
305  HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
306  if (MacroInfo *MI =
307  getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro))) {
308  MI->UsedForHeaderGuard = true;
309  }
310  if (const IdentifierInfo *DefinedMacro =
311  CurPPLexer->MIOpt.GetDefinedMacro()) {
312  if (!isMacroDefined(ControllingMacro) &&
313  DefinedMacro != ControllingMacro &&
314  HeaderInfo.FirstTimeLexingFile(FE)) {
315 
316  // If the edit distance between the two macros is more than 50%,
317  // DefinedMacro may not be header guard, or can be header guard of
318  // another header file. Therefore, it maybe defining something
319  // completely different. This can be observed in the wild when
320  // handling feature macros or header guards in different files.
321 
322  const StringRef ControllingMacroName = ControllingMacro->getName();
323  const StringRef DefinedMacroName = DefinedMacro->getName();
324  const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
325  DefinedMacroName.size()) / 2;
326  const unsigned ED = ControllingMacroName.edit_distance(
327  DefinedMacroName, true, MaxHalfLength);
328  if (ED <= MaxHalfLength) {
329  // Emit a warning for a bad header guard.
330  Diag(CurPPLexer->MIOpt.GetMacroLocation(),
331  diag::warn_header_guard)
332  << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
333  Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
334  diag::note_header_guard)
335  << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
336  << ControllingMacro
338  CurPPLexer->MIOpt.GetDefinedLocation(),
339  ControllingMacro->getName());
340  }
341  }
342  }
343  }
344  }
345  }
346 
347  // Complain about reaching a true EOF within arc_cf_code_audited.
348  // We don't want to complain about reaching the end of a macro
349  // instantiation or a _Pragma.
350  if (PragmaARCCFCodeAuditedLoc.isValid() &&
351  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
352  Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
353 
354  // Recover by leaving immediately.
355  PragmaARCCFCodeAuditedLoc = SourceLocation();
356  }
357 
358  // Complain about reaching a true EOF within assume_nonnull.
359  // We don't want to complain about reaching the end of a macro
360  // instantiation or a _Pragma.
361  if (PragmaAssumeNonNullLoc.isValid() &&
362  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
363  Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
364 
365  // Recover by leaving immediately.
366  PragmaAssumeNonNullLoc = SourceLocation();
367  }
368 
369  // If this is a #include'd file, pop it off the include stack and continue
370  // lexing the #includer file.
371  if (!IncludeMacroStack.empty()) {
372 
373  // If we lexed the code-completion file, act as if we reached EOF.
374  if (isCodeCompletionEnabled() && CurPPLexer &&
375  SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
376  CodeCompletionFileLoc) {
377  if (CurLexer) {
378  Result.startToken();
379  CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
380  CurLexer.reset();
381  } else {
382  assert(CurPTHLexer && "Got EOF but no current lexer set!");
383  CurPTHLexer->getEOF(Result);
384  CurPTHLexer.reset();
385  }
386 
387  CurPPLexer = nullptr;
388  return true;
389  }
390 
391  if (!isEndOfMacro && CurPPLexer &&
392  SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
393  // Notify SourceManager to record the number of FileIDs that were created
394  // during lexing of the #include'd file.
395  unsigned NumFIDs =
396  SourceMgr.local_sloc_entry_size() -
397  CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
398  SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
399  }
400 
401  FileID ExitedFID;
402  if (Callbacks && !isEndOfMacro && CurPPLexer)
403  ExitedFID = CurPPLexer->getFileID();
404 
405  bool LeavingSubmodule = CurSubmodule && CurLexer;
406  if (LeavingSubmodule) {
407  // Notify the parser that we've left the module.
408  const char *EndPos = getCurLexerEndPos();
409  Result.startToken();
410  CurLexer->BufferPtr = EndPos;
411  CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
412  Result.setAnnotationEndLoc(Result.getLocation());
413  Result.setAnnotationValue(CurSubmodule);
414 
415  // We're done with this submodule.
416  LeaveSubmodule();
417  }
418 
419  // We're done with the #included file.
421 
422  // Propagate info about start-of-line/leading white-space/etc.
423  PropagateLineStartLeadingSpaceInfo(Result);
424 
425  // Notify the client, if desired, that we are in a new source file.
426  if (Callbacks && !isEndOfMacro && CurPPLexer) {
427  SrcMgr::CharacteristicKind FileType =
428  SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
429  Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
430  PPCallbacks::ExitFile, FileType, ExitedFID);
431  }
432 
433  // Client should lex another token unless we generated an EOM.
434  return LeavingSubmodule;
435  }
436 
437  // If this is the end of the main file, form an EOF token.
438  if (CurLexer) {
439  const char *EndPos = getCurLexerEndPos();
440  Result.startToken();
441  CurLexer->BufferPtr = EndPos;
442  CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
443 
444  if (isCodeCompletionEnabled()) {
445  // Inserting the code-completion point increases the source buffer by 1,
446  // but the main FileID was created before inserting the point.
447  // Compensate by reducing the EOF location by 1, otherwise the location
448  // will point to the next FileID.
449  // FIXME: This is hacky, the code-completion point should probably be
450  // inserted before the main FileID is created.
451  if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
452  Result.setLocation(Result.getLocation().getLocWithOffset(-1));
453  }
454 
456  // We're done with lexing.
457  CurLexer.reset();
458  } else {
459  assert(CurPTHLexer && "Got EOF but no current lexer set!");
460  CurPTHLexer->getEOF(Result);
461  CurPTHLexer.reset();
462  }
463 
465  CurPPLexer = nullptr;
466 
467  if (TUKind == TU_Complete) {
468  // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
469  // collected all macro locations that we need to warn because they are not
470  // used.
472  I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
473  I!=E; ++I)
474  Diag(*I, diag::pp_macro_not_used);
475  }
476 
477  // If we are building a module that has an umbrella header, make sure that
478  // each of the headers within the directory covered by the umbrella header
479  // was actually included by the umbrella header.
480  if (Module *Mod = getCurrentModule()) {
481  if (Mod->getUmbrellaHeader()) {
482  SourceLocation StartLoc
483  = SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
484 
485  if (!getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
486  StartLoc)) {
488  const DirectoryEntry *Dir = Mod->getUmbrellaDir().Entry;
489  vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem();
490  std::error_code EC;
491  for (vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), End;
492  Entry != End && !EC; Entry.increment(EC)) {
493  using llvm::StringSwitch;
494 
495  // Check whether this entry has an extension typically associated with
496  // headers.
497  if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->getName()))
498  .Cases(".h", ".H", ".hh", ".hpp", true)
499  .Default(false))
500  continue;
501 
502  if (const FileEntry *Header =
503  getFileManager().getFile(Entry->getName()))
504  if (!getSourceManager().hasFileInfo(Header)) {
505  if (!ModMap.isHeaderInUnavailableModule(Header)) {
506  // Find the relative path that would access this header.
507  SmallString<128> RelativePath;
508  computeRelativePath(FileMgr, Dir, Header, RelativePath);
509  Diag(StartLoc, diag::warn_uncovered_module_header)
510  << Mod->getFullModuleName() << RelativePath;
511  }
512  }
513  }
514  }
515  }
516  }
517 
518  return true;
519 }
520 
521 /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
522 /// hits the end of its token stream.
524  assert(CurTokenLexer && !CurPPLexer &&
525  "Ending a macro when currently in a #include file!");
526 
527  if (!MacroExpandingLexersStack.empty() &&
528  MacroExpandingLexersStack.back().first == CurTokenLexer.get())
529  removeCachedMacroExpandedTokensOfLastLexer();
530 
531  // Delete or cache the now-dead macro expander.
532  if (NumCachedTokenLexers == TokenLexerCacheSize)
533  CurTokenLexer.reset();
534  else
535  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
536 
537  // Handle this like a #include file being popped off the stack.
538  return HandleEndOfFile(Result, true);
539 }
540 
541 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
542 /// lexer stack. This should only be used in situations where the current
543 /// state of the top-of-stack lexer is unknown.
545  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
546 
547  if (CurTokenLexer) {
548  // Delete or cache the now-dead macro expander.
549  if (NumCachedTokenLexers == TokenLexerCacheSize)
550  CurTokenLexer.reset();
551  else
552  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
553  }
554 
555  PopIncludeMacroStack();
556 }
557 
558 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
559 /// comment (/##/) in microsoft mode, this method handles updating the current
560 /// state, returning the token on the next source line.
562  assert(CurTokenLexer && !CurPPLexer &&
563  "Pasted comment can only be formed from macro");
564  // We handle this by scanning for the closest real lexer, switching it to
565  // raw mode and preprocessor mode. This will cause it to return \n as an
566  // explicit EOD token.
567  PreprocessorLexer *FoundLexer = nullptr;
568  bool LexerWasInPPMode = false;
569  for (unsigned i = 0, e = IncludeMacroStack.size(); i != e; ++i) {
570  IncludeStackInfo &ISI = *(IncludeMacroStack.end()-i-1);
571  if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
572 
573  // Once we find a real lexer, mark it as raw mode (disabling macro
574  // expansions) and preprocessor mode (return EOD). We know that the lexer
575  // was *not* in raw mode before, because the macro that the comment came
576  // from was expanded. However, it could have already been in preprocessor
577  // mode (#if COMMENT) in which case we have to return it to that mode and
578  // return EOD.
579  FoundLexer = ISI.ThePPLexer;
580  FoundLexer->LexingRawMode = true;
581  LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
582  FoundLexer->ParsingPreprocessorDirective = true;
583  break;
584  }
585 
586  // Okay, we either found and switched over the lexer, or we didn't find a
587  // lexer. In either case, finish off the macro the comment came from, getting
588  // the next token.
589  if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
590 
591  // Discarding comments as long as we don't have EOF or EOD. This 'comments
592  // out' the rest of the line, including any tokens that came from other macros
593  // that were active, as in:
594  // #define submacro a COMMENT b
595  // submacro c
596  // which should lex to 'a' only: 'b' and 'c' should be removed.
597  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
598  Lex(Tok);
599 
600  // If we got an eod token, then we successfully found the end of the line.
601  if (Tok.is(tok::eod)) {
602  assert(FoundLexer && "Can't get end of line without an active lexer");
603  // Restore the lexer back to normal mode instead of raw mode.
604  FoundLexer->LexingRawMode = false;
605 
606  // If the lexer was already in preprocessor mode, just return the EOD token
607  // to finish the preprocessor line.
608  if (LexerWasInPPMode) return;
609 
610  // Otherwise, switch out of PP mode and return the next lexed token.
611  FoundLexer->ParsingPreprocessorDirective = false;
612  return Lex(Tok);
613  }
614 
615  // If we got an EOF token, then we reached the end of the token stream but
616  // didn't find an explicit \n. This can only happen if there was no lexer
617  // active (an active lexer would return EOD at EOF if there was no \n in
618  // preprocessor directive mode), so just return EOF as our token.
619  assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
620 }
621 
622 void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc) {
623  if (!getLangOpts().ModulesLocalVisibility) {
624  // Just track that we entered this submodule.
625  BuildingSubmoduleStack.push_back(BuildingSubmoduleInfo(
626  M, ImportLoc, CurSubmoduleState, PendingModuleMacroNames.size()));
627  return;
628  }
629 
630  // Resolve as much of the module definition as we can now, before we enter
631  // one of its headers.
632  // FIXME: Can we enable Complain here?
633  // FIXME: Can we do this when local visibility is disabled?
635  ModMap.resolveExports(M, /*Complain=*/false);
636  ModMap.resolveUses(M, /*Complain=*/false);
637  ModMap.resolveConflicts(M, /*Complain=*/false);
638 
639  // If this is the first time we've entered this module, set up its state.
640  auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
641  auto &State = R.first->second;
642  bool FirstTime = R.second;
643  if (FirstTime) {
644  // Determine the set of starting macros for this submodule; take these
645  // from the "null" module (the predefines buffer).
646  //
647  // FIXME: If we have local visibility but not modules enabled, the
648  // NullSubmoduleState is polluted by #defines in the top-level source
649  // file.
650  auto &StartingMacros = NullSubmoduleState.Macros;
651 
652  // Restore to the starting state.
653  // FIXME: Do this lazily, when each macro name is first referenced.
654  for (auto &Macro : StartingMacros) {
655  // Skip uninteresting macros.
656  if (!Macro.second.getLatest() &&
657  Macro.second.getOverriddenMacros().empty())
658  continue;
659 
660  MacroState MS(Macro.second.getLatest());
661  MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
662  State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
663  }
664  }
665 
666  // Track that we entered this module.
667  BuildingSubmoduleStack.push_back(BuildingSubmoduleInfo(
668  M, ImportLoc, CurSubmoduleState, PendingModuleMacroNames.size()));
669 
670  // Switch to this submodule as the current submodule.
671  CurSubmoduleState = &State;
672 
673  // This module is visible to itself.
674  if (FirstTime)
675  makeModuleVisible(M, ImportLoc);
676 }
677 
678 bool Preprocessor::needModuleMacros() const {
679  // If we're not within a submodule, we never need to create ModuleMacros.
680  if (BuildingSubmoduleStack.empty())
681  return false;
682  // If we are tracking module macro visibility even for textually-included
683  // headers, we need ModuleMacros.
684  if (getLangOpts().ModulesLocalVisibility)
685  return true;
686  // Otherwise, we only need module macros if we're actually compiling a module
687  // interface.
688  return getLangOpts().CompilingModule;
689 }
690 
691 void Preprocessor::LeaveSubmodule() {
692  auto &Info = BuildingSubmoduleStack.back();
693 
694  Module *LeavingMod = Info.M;
695  SourceLocation ImportLoc = Info.ImportLoc;
696 
697  if (!needModuleMacros() ||
698  (!getLangOpts().ModulesLocalVisibility &&
699  LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
700  // If we don't need module macros, or this is not a module for which we
701  // are tracking macro visibility, don't build any, and preserve the list
702  // of pending names for the surrounding submodule.
703  BuildingSubmoduleStack.pop_back();
704  makeModuleVisible(LeavingMod, ImportLoc);
705  return;
706  }
707 
708  // Create ModuleMacros for any macros defined in this submodule.
709  llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
710  for (unsigned I = Info.OuterPendingModuleMacroNames;
711  I != PendingModuleMacroNames.size(); ++I) {
712  auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
713  if (!VisitedMacros.insert(II).second)
714  continue;
715 
716  auto MacroIt = CurSubmoduleState->Macros.find(II);
717  if (MacroIt == CurSubmoduleState->Macros.end())
718  continue;
719  auto &Macro = MacroIt->second;
720 
721  // Find the starting point for the MacroDirective chain in this submodule.
722  MacroDirective *OldMD = nullptr;
723  auto *OldState = Info.OuterSubmoduleState;
724  if (getLangOpts().ModulesLocalVisibility)
725  OldState = &NullSubmoduleState;
726  if (OldState && OldState != CurSubmoduleState) {
727  // FIXME: It'd be better to start at the state from when we most recently
728  // entered this submodule, but it doesn't really matter.
729  auto &OldMacros = OldState->Macros;
730  auto OldMacroIt = OldMacros.find(II);
731  if (OldMacroIt == OldMacros.end())
732  OldMD = nullptr;
733  else
734  OldMD = OldMacroIt->second.getLatest();
735  }
736 
737  // This module may have exported a new macro. If so, create a ModuleMacro
738  // representing that fact.
739  bool ExplicitlyPublic = false;
740  for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
741  assert(MD && "broken macro directive chain");
742 
743  // Stop on macros defined in other submodules of this module that we
744  // #included along the way. There's no point doing this if we're
745  // tracking local submodule visibility, since there can be no such
746  // directives in our list.
747  if (!getLangOpts().ModulesLocalVisibility) {
748  Module *Mod = getModuleContainingLocation(MD->getLocation());
749  if (Mod != LeavingMod &&
750  Mod->getTopLevelModule() == LeavingMod->getTopLevelModule())
751  break;
752  }
753 
754  if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
755  // The latest visibility directive for a name in a submodule affects
756  // all the directives that come before it.
757  if (VisMD->isPublic())
758  ExplicitlyPublic = true;
759  else if (!ExplicitlyPublic)
760  // Private with no following public directive: not exported.
761  break;
762  } else {
763  MacroInfo *Def = nullptr;
764  if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
765  Def = DefMD->getInfo();
766 
767  // FIXME: Issue a warning if multiple headers for the same submodule
768  // define a macro, rather than silently ignoring all but the first.
769  bool IsNew;
770  // Don't bother creating a module macro if it would represent a #undef
771  // that doesn't override anything.
772  if (Def || !Macro.getOverriddenMacros().empty())
773  addModuleMacro(LeavingMod, II, Def,
774  Macro.getOverriddenMacros(), IsNew);
775  break;
776  }
777  }
778  }
779  PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
780 
781  // FIXME: Before we leave this submodule, we should parse all the other
782  // headers within it. Otherwise, we're left with an inconsistent state
783  // where we've made the module visible but don't yet have its complete
784  // contents.
785 
786  // Put back the outer module's state, if we're tracking it.
787  if (getLangOpts().ModulesLocalVisibility)
788  CurSubmoduleState = Info.OuterSubmoduleState;
789 
790  BuildingSubmoduleStack.pop_back();
791 
792  // A nested #include makes the included submodule visible.
793  makeModuleVisible(LeavingMod, ImportLoc);
794 }
SourceManager & getSourceManager() const
Definition: Preprocessor.h:694
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:46
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:117
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
Defines the clang::FileManager interface and associated types.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
Definition: ModuleMap.cpp:432
Module * getTopLevelModule()
Retrieve the top-level module for this (sub)module, which may be this module.
Definition: Basic/Module.h:368
Defines the SourceManager interface.
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Definition: ModuleMap.cpp:888
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:78
ModuleMap & getModuleMap()
Retrieve the module map.
Definition: HeaderSearch.h:590
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
SourceLocation GetDefinedLocation() const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasFileInfo(const FileEntry *File) const
Module * getModuleContainingLocation(SourceLocation Loc)
Find the module that contains the specified location, either directly or indirectly.
The virtual file system interface.
const StringRef FilePath
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:418
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
Definition: ModuleMap.cpp:901
LineState State
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:858
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:690
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
Describes a module or submodule.
Definition: Basic/Module.h:47
FileManager & getFileManager() const
Definition: Preprocessor.h:693
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
IntrusiveRefCntPtr< vfs::FileSystem > getVirtualFileSystem() const
Definition: FileManager.h:226
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:695
const FileEntry * getFileEntry() const
getFileEntry - Return the FileEntry corresponding to this FileID.
detail::InMemoryDirectory::const_iterator I
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
std::string CurrentModule
The name of the current module, of which the main source file is a part.
Definition: LangOptions.h:107
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setAnnotationValue(void *val)
Definition: Token.h:227
bool LexingRawMode
True if in raw mode.
void makeModuleVisible(Module *M, SourceLocation Loc)
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
Defines the clang::Preprocessor interface.
const char * getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
Definition: ModuleMap.cpp:875
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
bool isNot(tok::TokenKind K) const
Definition: Token.h:95
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location...
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
Definition: Basic/Module.h:379
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:307
const char * getName() const
Definition: FileManager.h:85
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
const TemplateArgument * iterator
Definition: Type.h:4233
bool isValid() const
Return true if this is a valid SourceLocation object.
void setAnnotationEndLoc(SourceLocation L)
Definition: Token.h:141
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:53
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
FileID getMainFileID() const
Returns the FileID of the main source file.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
bool isIgnored(unsigned DiagID, SourceLocation Loc) const
Determine whether the diagnostic is known to be ignored.
Definition: Diagnostic.h:652
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:687
const char * getName() const
Definition: FileManager.h:45
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:793
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined...
detail::InMemoryDirectory::const_iterator E
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:34
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:40
char __ovld __cnfn max(char x, char y)
Returns y if x < y, otherwise it returns x.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
Definition: Diagnostic.h:115
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs) const
Set the number of FileIDs (files and macros) that were created during preprocessing of FID...
The translation unit is a complete translation unit.
Definition: LangOptions.h:174
SourceLocation GetMacroLocation() const
bool FirstTimeLexingFile(const FileEntry *File)
Return true if this is the first time encountering this header.
Definition: HeaderSearch.h:452
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
void setLocation(SourceLocation L)
Definition: Token.h:131
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:95
void startToken()
Reset all flags to cleared.
Definition: Token.h:168
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:342
void SetFileControllingMacro(const FileEntry *File, const IdentifierInfo *ControllingMacro)
Mark the specified file as having a controlling macro.
Definition: HeaderSearch.h:446
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...