clang  3.9.0
GlobalModuleIndex.cpp
Go to the documentation of this file.
1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the GlobalModuleIndex class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ASTReaderInternals.h"
17 #include "clang/Lex/HeaderSearch.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/Bitcode/BitstreamReader.h"
26 #include "llvm/Bitcode/BitstreamWriter.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/LockFileManager.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/OnDiskHashTable.h"
31 #include "llvm/Support/Path.h"
32 #include <cstdio>
33 using namespace clang;
34 using namespace serialization;
35 
36 //----------------------------------------------------------------------------//
37 // Shared constants
38 //----------------------------------------------------------------------------//
39 namespace {
40  enum {
41  /// \brief The block containing the index.
42  GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
43  };
44 
45  /// \brief Describes the record types in the index.
47  /// \brief Contains version information and potentially other metadata,
48  /// used to determine if we can read this global index file.
49  INDEX_METADATA,
50  /// \brief Describes a module, including its file name and dependencies.
51  MODULE,
52  /// \brief The index for identifiers.
53  IDENTIFIER_INDEX
54  };
55 }
56 
57 /// \brief The name of the global index file.
58 static const char * const IndexFileName = "modules.idx";
59 
60 /// \brief The global index file version.
61 static const unsigned CurrentVersion = 1;
62 
63 //----------------------------------------------------------------------------//
64 // Global module index reader.
65 //----------------------------------------------------------------------------//
66 
67 namespace {
68 
69 /// \brief Trait used to read the identifier index from the on-disk hash
70 /// table.
71 class IdentifierIndexReaderTrait {
72 public:
73  typedef StringRef external_key_type;
74  typedef StringRef internal_key_type;
75  typedef SmallVector<unsigned, 2> data_type;
76  typedef unsigned hash_value_type;
77  typedef unsigned offset_type;
78 
79  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
80  return a == b;
81  }
82 
83  static hash_value_type ComputeHash(const internal_key_type& a) {
84  return llvm::HashString(a);
85  }
86 
87  static std::pair<unsigned, unsigned>
88  ReadKeyDataLength(const unsigned char*& d) {
89  using namespace llvm::support;
90  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
91  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
92  return std::make_pair(KeyLen, DataLen);
93  }
94 
95  static const internal_key_type&
96  GetInternalKey(const external_key_type& x) { return x; }
97 
98  static const external_key_type&
99  GetExternalKey(const internal_key_type& x) { return x; }
100 
101  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
102  return StringRef((const char *)d, n);
103  }
104 
105  static data_type ReadData(const internal_key_type& k,
106  const unsigned char* d,
107  unsigned DataLen) {
108  using namespace llvm::support;
109 
110  data_type Result;
111  while (DataLen > 0) {
112  unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
113  Result.push_back(ID);
114  DataLen -= 4;
115  }
116 
117  return Result;
118  }
119 };
120 
122  IdentifierIndexTable;
123 
124 }
125 
126 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
127  llvm::BitstreamCursor Cursor)
128  : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
129  NumIdentifierLookupHits() {
130  // Read the global index.
131  bool InGlobalIndexBlock = false;
132  bool Done = false;
133  while (!Done) {
134  llvm::BitstreamEntry Entry = Cursor.advance();
135 
136  switch (Entry.Kind) {
138  return;
139 
140  case llvm::BitstreamEntry::EndBlock:
141  if (InGlobalIndexBlock) {
142  InGlobalIndexBlock = false;
143  Done = true;
144  continue;
145  }
146  return;
147 
148 
149  case llvm::BitstreamEntry::Record:
150  // Entries in the global index block are handled below.
151  if (InGlobalIndexBlock)
152  break;
153 
154  return;
155 
156  case llvm::BitstreamEntry::SubBlock:
157  if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158  if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
159  return;
160 
161  InGlobalIndexBlock = true;
162  } else if (Cursor.SkipBlock()) {
163  return;
164  }
165  continue;
166  }
167 
169  StringRef Blob;
170  switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
171  case INDEX_METADATA:
172  // Make sure that the version matches.
173  if (Record.size() < 1 || Record[0] != CurrentVersion)
174  return;
175  break;
176 
177  case MODULE: {
178  unsigned Idx = 0;
179  unsigned ID = Record[Idx++];
180 
181  // Make room for this module's information.
182  if (ID == Modules.size())
183  Modules.push_back(ModuleInfo());
184  else
185  Modules.resize(ID + 1);
186 
187  // Size/modification time for this module file at the time the
188  // global index was built.
189  Modules[ID].Size = Record[Idx++];
190  Modules[ID].ModTime = Record[Idx++];
191 
192  // File name.
193  unsigned NameLen = Record[Idx++];
194  Modules[ID].FileName.assign(Record.begin() + Idx,
195  Record.begin() + Idx + NameLen);
196  Idx += NameLen;
197 
198  // Dependencies
199  unsigned NumDeps = Record[Idx++];
200  Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201  Record.begin() + Idx,
202  Record.begin() + Idx + NumDeps);
203  Idx += NumDeps;
204 
205  // Make sure we're at the end of the record.
206  assert(Idx == Record.size() && "More module info?");
207 
208  // Record this module as an unresolved module.
209  // FIXME: this doesn't work correctly for module names containing path
210  // separators.
211  StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
212  // Remove the -<hash of ModuleMapPath>
213  ModuleName = ModuleName.rsplit('-').first;
214  UnresolvedModules[ModuleName] = ID;
215  break;
216  }
217 
218  case IDENTIFIER_INDEX:
219  // Wire up the identifier index.
220  if (Record[0]) {
221  IdentifierIndex = IdentifierIndexTable::Create(
222  (const unsigned char *)Blob.data() + Record[0],
223  (const unsigned char *)Blob.data() + sizeof(uint32_t),
224  (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
225  }
226  break;
227  }
228  }
229 }
230 
231 GlobalModuleIndex::~GlobalModuleIndex() {
232  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
233 }
234 
235 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
237  // Load the index file, if it's there.
238  llvm::SmallString<128> IndexPath;
239  IndexPath += Path;
240  llvm::sys::path::append(IndexPath, IndexFileName);
241 
242  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
243  llvm::MemoryBuffer::getFile(IndexPath.c_str());
244  if (!BufferOrErr)
245  return std::make_pair(nullptr, EC_NotFound);
246  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
247 
248  /// \brief The bitstream reader from which we'll read the AST file.
249  llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
250  (const unsigned char *)Buffer->getBufferEnd());
251 
252  /// \brief The main bitstream cursor for the main block.
253  llvm::BitstreamCursor Cursor(Reader);
254 
255  // Sniff for the signature.
256  if (Cursor.Read(8) != 'B' ||
257  Cursor.Read(8) != 'C' ||
258  Cursor.Read(8) != 'G' ||
259  Cursor.Read(8) != 'I') {
260  return std::make_pair(nullptr, EC_IOError);
261  }
262 
263  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
264  EC_None);
265 }
266 
267 void
269  ModuleFiles.clear();
270  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
271  if (ModuleFile *MF = Modules[I].File)
272  ModuleFiles.push_back(MF);
273  }
274 }
275 
277  ModuleFile *File,
278  SmallVectorImpl<ModuleFile *> &Dependencies) {
279  // Look for information about this module file.
281  = ModulesByFile.find(File);
282  if (Known == ModulesByFile.end())
283  return;
284 
285  // Record dependencies.
286  Dependencies.clear();
287  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
288  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
289  if (ModuleFile *MF = Modules[I].File)
290  Dependencies.push_back(MF);
291  }
292 }
293 
295  Hits.clear();
296 
297  // If there's no identifier index, there is nothing we can do.
298  if (!IdentifierIndex)
299  return false;
300 
301  // Look into the identifier index.
302  ++NumIdentifierLookups;
303  IdentifierIndexTable &Table
304  = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
305  IdentifierIndexTable::iterator Known = Table.find(Name);
306  if (Known == Table.end()) {
307  return true;
308  }
309 
310  SmallVector<unsigned, 2> ModuleIDs = *Known;
311  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
312  if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
313  Hits.insert(MF);
314  }
315 
316  ++NumIdentifierLookupHits;
317  return true;
318 }
319 
320 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
321  // Look for the module in the global module index based on the module name.
322  StringRef Name = File->ModuleName;
323  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
324  if (Known == UnresolvedModules.end()) {
325  return true;
326  }
327 
328  // Rectify this module with the global module index.
329  ModuleInfo &Info = Modules[Known->second];
330 
331  // If the size and modification time match what we expected, record this
332  // module file.
333  bool Failed = true;
334  if (File->File->getSize() == Info.Size &&
335  File->File->getModificationTime() == Info.ModTime) {
336  Info.File = File;
337  ModulesByFile[File] = Known->second;
338 
339  Failed = false;
340  }
341 
342  // One way or another, we have resolved this module file.
343  UnresolvedModules.erase(Known);
344  return Failed;
345 }
346 
348  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
349  if (NumIdentifierLookups) {
350  fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
351  NumIdentifierLookupHits, NumIdentifierLookups,
352  (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
353  }
354  std::fprintf(stderr, "\n");
355 }
356 
357 LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
358  llvm::errs() << "*** Global Module Index Dump:\n";
359  llvm::errs() << "Module files:\n";
360  for (auto &MI : Modules) {
361  llvm::errs() << "** " << MI.FileName << "\n";
362  if (MI.File)
363  MI.File->dump();
364  else
365  llvm::errs() << "\n";
366  }
367  llvm::errs() << "\n";
368 }
369 
370 //----------------------------------------------------------------------------//
371 // Global module index writer.
372 //----------------------------------------------------------------------------//
373 
374 namespace {
375  /// \brief Provides information about a specific module file.
376  struct ModuleFileInfo {
377  /// \brief The numberic ID for this module file.
378  unsigned ID;
379 
380  /// \brief The set of modules on which this module depends. Each entry is
381  /// a module ID.
382  SmallVector<unsigned, 4> Dependencies;
383  };
384 
385  /// \brief Builder that generates the global module index file.
386  class GlobalModuleIndexBuilder {
387  FileManager &FileMgr;
388  const PCHContainerReader &PCHContainerRdr;
389 
390  /// \brief Mapping from files to module file information.
391  typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
392 
393  /// \brief Information about each of the known module files.
394  ModuleFilesMap ModuleFiles;
395 
396  /// \brief Mapping from identifiers to the list of module file IDs that
397  /// consider this identifier to be interesting.
398  typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
399 
400  /// \brief A mapping from all interesting identifiers to the set of module
401  /// files in which those identifiers are considered interesting.
402  InterestingIdentifierMap InterestingIdentifiers;
403 
404  /// \brief Write the block-info block for the global module index file.
405  void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
406 
407  /// \brief Retrieve the module file information for the given file.
408  ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
410  = ModuleFiles.find(File);
411  if (Known != ModuleFiles.end())
412  return Known->second;
413 
414  unsigned NewID = ModuleFiles.size();
415  ModuleFileInfo &Info = ModuleFiles[File];
416  Info.ID = NewID;
417  return Info;
418  }
419 
420  public:
421  explicit GlobalModuleIndexBuilder(
422  FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
423  : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
424 
425  /// \brief Load the contents of the given module file into the builder.
426  ///
427  /// \returns true if an error occurred, false otherwise.
428  bool loadModuleFile(const FileEntry *File);
429 
430  /// \brief Write the index to the given bitstream.
431  void writeIndex(llvm::BitstreamWriter &Stream);
432  };
433 }
434 
435 static void emitBlockID(unsigned ID, const char *Name,
436  llvm::BitstreamWriter &Stream,
437  SmallVectorImpl<uint64_t> &Record) {
438  Record.clear();
439  Record.push_back(ID);
440  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
441 
442  // Emit the block name if present.
443  if (!Name || Name[0] == 0) return;
444  Record.clear();
445  while (*Name)
446  Record.push_back(*Name++);
447  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
448 }
449 
450 static void emitRecordID(unsigned ID, const char *Name,
451  llvm::BitstreamWriter &Stream,
452  SmallVectorImpl<uint64_t> &Record) {
453  Record.clear();
454  Record.push_back(ID);
455  while (*Name)
456  Record.push_back(*Name++);
457  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
458 }
459 
460 void
461 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
463  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
464 
465 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
466 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
467  BLOCK(GLOBAL_INDEX_BLOCK);
468  RECORD(INDEX_METADATA);
469  RECORD(MODULE);
470  RECORD(IDENTIFIER_INDEX);
471 #undef RECORD
472 #undef BLOCK
473 
474  Stream.ExitBlock();
475 }
476 
477 namespace {
478  class InterestingASTIdentifierLookupTrait
480 
481  public:
482  /// \brief The identifier and whether it is "interesting".
483  typedef std::pair<StringRef, bool> data_type;
484 
485  data_type ReadData(const internal_key_type& k,
486  const unsigned char* d,
487  unsigned DataLen) {
488  // The first bit indicates whether this identifier is interesting.
489  // That's all we care about.
490  using namespace llvm::support;
491  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
492  bool IsInteresting = RawID & 0x01;
493  return std::make_pair(k, IsInteresting);
494  }
495  };
496 }
497 
498 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
499  // Open the module file.
500 
501  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
502  if (!Buffer) {
503  return true;
504  }
505 
506  // Initialize the input stream
507  llvm::BitstreamReader InStreamFile;
508  PCHContainerRdr.ExtractPCH((*Buffer)->getMemBufferRef(), InStreamFile);
509  llvm::BitstreamCursor InStream(InStreamFile);
510 
511  // Sniff for the signature.
512  if (InStream.Read(8) != 'C' ||
513  InStream.Read(8) != 'P' ||
514  InStream.Read(8) != 'C' ||
515  InStream.Read(8) != 'H') {
516  return true;
517  }
518 
519  // Record this module file and assign it a unique ID (if it doesn't have
520  // one already).
521  unsigned ID = getModuleFileInfo(File).ID;
522 
523  // Search for the blocks and records we care about.
524  enum { Other, ControlBlock, ASTBlock } State = Other;
525  bool Done = false;
526  while (!Done) {
527  llvm::BitstreamEntry Entry = InStream.advance();
528  switch (Entry.Kind) {
530  Done = true;
531  continue;
532 
533  case llvm::BitstreamEntry::Record:
534  // In the 'other' state, just skip the record. We don't care.
535  if (State == Other) {
536  InStream.skipRecord(Entry.ID);
537  continue;
538  }
539 
540  // Handle potentially-interesting records below.
541  break;
542 
543  case llvm::BitstreamEntry::SubBlock:
544  if (Entry.ID == CONTROL_BLOCK_ID) {
545  if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
546  return true;
547 
548  // Found the control block.
549  State = ControlBlock;
550  continue;
551  }
552 
553  if (Entry.ID == AST_BLOCK_ID) {
554  if (InStream.EnterSubBlock(AST_BLOCK_ID))
555  return true;
556 
557  // Found the AST block.
558  State = ASTBlock;
559  continue;
560  }
561 
562  if (InStream.SkipBlock())
563  return true;
564 
565  continue;
566 
567  case llvm::BitstreamEntry::EndBlock:
568  State = Other;
569  continue;
570  }
571 
572  // Read the given record.
574  StringRef Blob;
575  unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
576 
577  // Handle module dependencies.
578  if (State == ControlBlock && Code == IMPORTS) {
579  // Load each of the imported PCH files.
580  unsigned Idx = 0, N = Record.size();
581  while (Idx < N) {
582  // Read information about the AST file.
583 
584  // Skip the imported kind
585  ++Idx;
586 
587  // Skip the import location
588  ++Idx;
589 
590  // Load stored size/modification time.
591  off_t StoredSize = (off_t)Record[Idx++];
592  time_t StoredModTime = (time_t)Record[Idx++];
593 
594  // Skip the stored signature.
595  // FIXME: we could read the signature out of the import and validate it.
596  Idx++;
597 
598  // Retrieve the imported file name.
599  unsigned Length = Record[Idx++];
600  SmallString<128> ImportedFile(Record.begin() + Idx,
601  Record.begin() + Idx + Length);
602  Idx += Length;
603 
604  // Find the imported module file.
605  const FileEntry *DependsOnFile
606  = FileMgr.getFile(ImportedFile, /*openFile=*/false,
607  /*cacheFailure=*/false);
608  if (!DependsOnFile ||
609  (StoredSize != DependsOnFile->getSize()) ||
610  (StoredModTime != DependsOnFile->getModificationTime()))
611  return true;
612 
613  // Record the dependency.
614  unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
615  getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
616  }
617 
618  continue;
619  }
620 
621  // Handle the identifier table
622  if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
624  InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
625  std::unique_ptr<InterestingIdentifierTable> Table(
627  (const unsigned char *)Blob.data() + Record[0],
628  (const unsigned char *)Blob.data() + sizeof(uint32_t),
629  (const unsigned char *)Blob.data()));
630  for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
631  DEnd = Table->data_end();
632  D != DEnd; ++D) {
633  std::pair<StringRef, bool> Ident = *D;
634  if (Ident.second)
635  InterestingIdentifiers[Ident.first].push_back(ID);
636  else
637  (void)InterestingIdentifiers[Ident.first];
638  }
639  }
640 
641  // We don't care about this record.
642  }
643 
644  return false;
645 }
646 
647 namespace {
648 
649 /// \brief Trait used to generate the identifier index as an on-disk hash
650 /// table.
651 class IdentifierIndexWriterTrait {
652 public:
653  typedef StringRef key_type;
654  typedef StringRef key_type_ref;
655  typedef SmallVector<unsigned, 2> data_type;
656  typedef const SmallVector<unsigned, 2> &data_type_ref;
657  typedef unsigned hash_value_type;
658  typedef unsigned offset_type;
659 
660  static hash_value_type ComputeHash(key_type_ref Key) {
661  return llvm::HashString(Key);
662  }
663 
664  std::pair<unsigned,unsigned>
665  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
666  using namespace llvm::support;
667  endian::Writer<little> LE(Out);
668  unsigned KeyLen = Key.size();
669  unsigned DataLen = Data.size() * 4;
670  LE.write<uint16_t>(KeyLen);
671  LE.write<uint16_t>(DataLen);
672  return std::make_pair(KeyLen, DataLen);
673  }
674 
675  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
676  Out.write(Key.data(), KeyLen);
677  }
678 
679  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
680  unsigned DataLen) {
681  using namespace llvm::support;
682  for (unsigned I = 0, N = Data.size(); I != N; ++I)
683  endian::Writer<little>(Out).write<uint32_t>(Data[I]);
684  }
685 };
686 
687 }
688 
689 void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
690  using namespace llvm;
691 
692  // Emit the file header.
693  Stream.Emit((unsigned)'B', 8);
694  Stream.Emit((unsigned)'C', 8);
695  Stream.Emit((unsigned)'G', 8);
696  Stream.Emit((unsigned)'I', 8);
697 
698  // Write the block-info block, which describes the records in this bitcode
699  // file.
700  emitBlockInfoBlock(Stream);
701 
702  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
703 
704  // Write the metadata.
706  Record.push_back(CurrentVersion);
707  Stream.EmitRecord(INDEX_METADATA, Record);
708 
709  // Write the set of known module files.
710  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
711  MEnd = ModuleFiles.end();
712  M != MEnd; ++M) {
713  Record.clear();
714  Record.push_back(M->second.ID);
715  Record.push_back(M->first->getSize());
716  Record.push_back(M->first->getModificationTime());
717 
718  // File name
719  StringRef Name(M->first->getName());
720  Record.push_back(Name.size());
721  Record.append(Name.begin(), Name.end());
722 
723  // Dependencies
724  Record.push_back(M->second.Dependencies.size());
725  Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
726  Stream.EmitRecord(MODULE, Record);
727  }
728 
729  // Write the identifier -> module file mapping.
730  {
731  llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
732  IdentifierIndexWriterTrait Trait;
733 
734  // Populate the hash table.
735  for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
736  IEnd = InterestingIdentifiers.end();
737  I != IEnd; ++I) {
738  Generator.insert(I->first(), I->second, Trait);
739  }
740 
741  // Create the on-disk hash table in a buffer.
743  uint32_t BucketOffset;
744  {
745  using namespace llvm::support;
746  llvm::raw_svector_ostream Out(IdentifierTable);
747  // Make sure that no bucket is at offset 0
748  endian::Writer<little>(Out).write<uint32_t>(0);
749  BucketOffset = Generator.Emit(Out, Trait);
750  }
751 
752  // Create a blob abbreviation
753  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
754  Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
755  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
756  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
757  unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
758 
759  // Write the identifier table
760  uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
761  Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
762  }
763 
764  Stream.ExitBlock();
765 }
766 
769  const PCHContainerReader &PCHContainerRdr,
770  StringRef Path) {
771  llvm::SmallString<128> IndexPath;
772  IndexPath += Path;
773  llvm::sys::path::append(IndexPath, IndexFileName);
774 
775  // Coordinate building the global index file with other processes that might
776  // try to do the same.
777  llvm::LockFileManager Locked(IndexPath);
778  switch (Locked) {
779  case llvm::LockFileManager::LFS_Error:
780  return EC_IOError;
781 
782  case llvm::LockFileManager::LFS_Owned:
783  // We're responsible for building the index ourselves. Do so below.
784  break;
785 
786  case llvm::LockFileManager::LFS_Shared:
787  // Someone else is responsible for building the index. We don't care
788  // when they finish, so we're done.
789  return EC_Building;
790  }
791 
792  // The module index builder.
793  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
794 
795  // Load each of the module files.
796  std::error_code EC;
797  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
798  D != DEnd && !EC;
799  D.increment(EC)) {
800  // If this isn't a module file, we don't care.
801  if (llvm::sys::path::extension(D->path()) != ".pcm") {
802  // ... unless it's a .pcm.lock file, which indicates that someone is
803  // in the process of rebuilding a module. They'll rebuild the index
804  // at the end of that translation unit, so we don't have to.
805  if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
806  return EC_Building;
807 
808  continue;
809  }
810 
811  // If we can't find the module file, skip it.
812  const FileEntry *ModuleFile = FileMgr.getFile(D->path());
813  if (!ModuleFile)
814  continue;
815 
816  // Load this module file.
817  if (Builder.loadModuleFile(ModuleFile))
818  return EC_IOError;
819  }
820 
821  // The output buffer, into which the global index will be written.
822  SmallVector<char, 16> OutputBuffer;
823  {
824  llvm::BitstreamWriter OutputStream(OutputBuffer);
825  Builder.writeIndex(OutputStream);
826  }
827 
828  // Write the global index file to a temporary file.
829  llvm::SmallString<128> IndexTmpPath;
830  int TmpFD;
831  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
832  IndexTmpPath))
833  return EC_IOError;
834 
835  // Open the temporary global index file for output.
836  llvm::raw_fd_ostream Out(TmpFD, true);
837  if (Out.has_error())
838  return EC_IOError;
839 
840  // Write the index.
841  Out.write(OutputBuffer.data(), OutputBuffer.size());
842  Out.close();
843  if (Out.has_error())
844  return EC_IOError;
845 
846  // Remove the old index file. It isn't relevant any more.
847  llvm::sys::fs::remove(IndexPath);
848 
849  // Rename the newly-written index file to the proper name.
850  if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
851  // Rename failed; just remove the
852  llvm::sys::fs::remove(IndexTmpPath);
853  return EC_IOError;
854  }
855 
856  // We're done.
857  return EC_None;
858 }
859 
860 namespace {
861  class GlobalIndexIdentifierIterator : public IdentifierIterator {
862  /// \brief The current position within the identifier lookup table.
863  IdentifierIndexTable::key_iterator Current;
864 
865  /// \brief The end position within the identifier lookup table.
866  IdentifierIndexTable::key_iterator End;
867 
868  public:
869  explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
870  Current = Idx.key_begin();
871  End = Idx.key_end();
872  }
873 
874  StringRef Next() override {
875  if (Current == End)
876  return StringRef();
877 
878  StringRef Result = *Current;
879  ++Current;
880  return Result;
881  }
882  };
883 }
884 
886  IdentifierIndexTable &Table =
887  *static_cast<IdentifierIndexTable *>(IdentifierIndex);
888  return new GlobalIndexIdentifierIterator(Table);
889 }
unsigned Length
void getKnownModules(SmallVectorImpl< ModuleFile * > &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:117
Defines the clang::FileManager interface and associated types.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void getModuleDependencies(ModuleFile *File, SmallVectorImpl< ModuleFile * > &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
static const unsigned CurrentVersion
The global index file version.
off_t getSize() const
Definition: FileManager.h:88
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
Definition: Type.h:4549
LineState State
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
detail::InMemoryDirectory::const_iterator I
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, StringRef Path)
Write a global index into the given.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
The result type of a method or function.
Record code for the identifier table.
Definition: ASTBitCodes.h:394
IndexRecordTypes
Describes the record types in the index.
StringRef FileName
Definition: Format.cpp:1313
SmallVectorImpl< AnnotatedLine * >::const_iterator Next
const TemplateArgument * iterator
Definition: Type.h:4233
const std::string ID
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:53
The AST block, which acts as a container around the full AST block.
Definition: ASTBitCodes.h:213
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:167
Record code for the list of other AST files imported by this AST file.
Definition: ASTBitCodes.h:266
There was an unspecified I/O error reading or writing the index.
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(StringRef Path)
Read a global index file for the given directory.
#define BLOCK(X)
time_t getModificationTime() const
Definition: FileManager.h:92
FormatToken * Current
BoundNodesTreeBuilder *const Builder
Dump information about a module file.
#define RECORD(X)
The control block, which contains all of the information that needs to be validated prior to committi...
Definition: ASTBitCodes.h:239
bool lookupIdentifier(StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.