21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/Bitcode/BitstreamReader.h"
26 #include "llvm/Bitcode/BitstreamWriter.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/LockFileManager.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/OnDiskHashTable.h"
31 #include "llvm/Support/Path.h"
33 using namespace clang;
34 using namespace serialization;
42 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
71 class IdentifierIndexReaderTrait {
73 typedef StringRef external_key_type;
74 typedef StringRef internal_key_type;
76 typedef unsigned hash_value_type;
77 typedef unsigned offset_type;
79 static bool EqualKey(
const internal_key_type& a,
const internal_key_type& b) {
83 static hash_value_type
ComputeHash(
const internal_key_type& a) {
84 return llvm::HashString(a);
87 static std::pair<unsigned, unsigned>
88 ReadKeyDataLength(
const unsigned char*& d) {
89 using namespace llvm::support;
90 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
91 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
92 return std::make_pair(KeyLen, DataLen);
95 static const internal_key_type&
96 GetInternalKey(
const external_key_type& x) {
return x; }
98 static const external_key_type&
99 GetExternalKey(
const internal_key_type& x) {
return x; }
101 static internal_key_type ReadKey(
const unsigned char* d,
unsigned n) {
102 return StringRef((
const char *)d, n);
105 static data_type ReadData(
const internal_key_type& k,
106 const unsigned char* d,
108 using namespace llvm::support;
111 while (DataLen > 0) {
112 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
122 IdentifierIndexTable;
126 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer>
Buffer,
127 llvm::BitstreamCursor
Cursor)
128 : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
129 NumIdentifierLookupHits() {
131 bool InGlobalIndexBlock =
false;
134 llvm::BitstreamEntry Entry = Cursor.advance();
136 switch (Entry.Kind) {
140 case llvm::BitstreamEntry::EndBlock:
141 if (InGlobalIndexBlock) {
142 InGlobalIndexBlock =
false;
149 case llvm::BitstreamEntry::Record:
151 if (InGlobalIndexBlock)
156 case llvm::BitstreamEntry::SubBlock:
157 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158 if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
161 InGlobalIndexBlock =
true;
162 }
else if (Cursor.SkipBlock()) {
179 unsigned ID = Record[Idx++];
182 if (ID == Modules.size())
183 Modules.push_back(ModuleInfo());
185 Modules.resize(ID + 1);
189 Modules[
ID].Size = Record[Idx++];
190 Modules[
ID].ModTime = Record[Idx++];
193 unsigned NameLen = Record[Idx++];
194 Modules[
ID].FileName.assign(Record.begin() + Idx,
195 Record.begin() + Idx + NameLen);
199 unsigned NumDeps = Record[Idx++];
200 Modules[
ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201 Record.begin() + Idx,
202 Record.begin() + Idx + NumDeps);
206 assert(Idx == Record.size() &&
"More module info?");
211 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].
FileName);
213 ModuleName = ModuleName.rsplit(
'-').first;
214 UnresolvedModules[ModuleName] =
ID;
218 case IDENTIFIER_INDEX:
222 (
const unsigned char *)Blob.data() + Record[0],
223 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
224 (
const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
231 GlobalModuleIndex::~GlobalModuleIndex() {
232 delete static_cast<IdentifierIndexTable *
>(IdentifierIndex);
235 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
242 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
243 llvm::MemoryBuffer::getFile(IndexPath.c_str());
246 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
249 llvm::BitstreamReader Reader((
const unsigned char *)Buffer->getBufferStart(),
250 (
const unsigned char *)Buffer->getBufferEnd());
253 llvm::BitstreamCursor
Cursor(Reader);
256 if (Cursor.Read(8) !=
'B' ||
257 Cursor.Read(8) !=
'C' ||
258 Cursor.Read(8) !=
'G' ||
259 Cursor.Read(8) !=
'I') {
270 for (
unsigned I = 0, N = Modules.size();
I != N; ++
I) {
271 if (ModuleFile *MF = Modules[
I].File)
272 ModuleFiles.push_back(MF);
281 = ModulesByFile.find(File);
282 if (Known == ModulesByFile.end())
286 Dependencies.clear();
288 for (
unsigned I = 0, N = StoredDependencies.size();
I != N; ++
I) {
289 if (ModuleFile *MF = Modules[
I].File)
290 Dependencies.push_back(MF);
298 if (!IdentifierIndex)
302 ++NumIdentifierLookups;
303 IdentifierIndexTable &Table
304 = *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
306 if (Known == Table.end()) {
311 for (
unsigned I = 0, N = ModuleIDs.size();
I != N; ++
I) {
312 if (ModuleFile *MF = Modules[ModuleIDs[
I]].File)
316 ++NumIdentifierLookupHits;
322 StringRef
Name = File->ModuleName;
324 if (Known == UnresolvedModules.end()) {
329 ModuleInfo &Info = Modules[Known->second];
334 if (File->File->getSize() == Info.Size &&
335 File->File->getModificationTime() == Info.ModTime) {
337 ModulesByFile[File] = Known->second;
343 UnresolvedModules.erase(Known);
348 std::fprintf(stderr,
"*** Global Module Index Statistics:\n");
349 if (NumIdentifierLookups) {
350 fprintf(stderr,
" %u / %u identifier lookups succeeded (%f%%)\n",
351 NumIdentifierLookupHits, NumIdentifierLookups,
352 (
double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
354 std::fprintf(stderr,
"\n");
358 llvm::errs() <<
"*** Global Module Index Dump:\n";
359 llvm::errs() <<
"Module files:\n";
360 for (
auto &MI : Modules) {
361 llvm::errs() <<
"** " << MI.FileName <<
"\n";
365 llvm::errs() <<
"\n";
367 llvm::errs() <<
"\n";
386 class GlobalModuleIndexBuilder {
391 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
394 ModuleFilesMap ModuleFiles;
398 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
402 InterestingIdentifierMap InterestingIdentifiers;
405 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
410 = ModuleFiles.find(File);
411 if (Known != ModuleFiles.end())
412 return Known->second;
414 unsigned NewID = ModuleFiles.size();
421 explicit GlobalModuleIndexBuilder(
423 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
428 bool loadModuleFile(
const FileEntry *File);
431 void writeIndex(llvm::BitstreamWriter &Stream);
436 llvm::BitstreamWriter &Stream,
439 Record.push_back(ID);
440 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
443 if (!Name || Name[0] == 0)
return;
446 Record.push_back(*Name++);
447 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
451 llvm::BitstreamWriter &Stream,
454 Record.push_back(ID);
456 Record.push_back(*Name++);
457 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
461 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
463 Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
465 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
466 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
467 BLOCK(GLOBAL_INDEX_BLOCK);
478 class InterestingASTIdentifierLookupTrait
483 typedef std::pair<StringRef, bool> data_type;
485 data_type ReadData(
const internal_key_type& k,
486 const unsigned char* d,
490 using namespace llvm::support;
491 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
492 bool IsInteresting = RawID & 0x01;
493 return std::make_pair(k, IsInteresting);
498 bool GlobalModuleIndexBuilder::loadModuleFile(
const FileEntry *File) {
501 auto Buffer = FileMgr.getBufferForFile(File,
true);
507 llvm::BitstreamReader InStreamFile;
508 PCHContainerRdr.ExtractPCH((*Buffer)->getMemBufferRef(), InStreamFile);
509 llvm::BitstreamCursor InStream(InStreamFile);
512 if (InStream.Read(8) !=
'C' ||
513 InStream.Read(8) !=
'P' ||
514 InStream.Read(8) !=
'C' ||
515 InStream.Read(8) !=
'H') {
521 unsigned ID = getModuleFileInfo(File).ID;
524 enum { Other, ControlBlock, ASTBlock }
State = Other;
527 llvm::BitstreamEntry Entry = InStream.advance();
528 switch (Entry.Kind) {
533 case llvm::BitstreamEntry::Record:
535 if (
State == Other) {
536 InStream.skipRecord(Entry.ID);
543 case llvm::BitstreamEntry::SubBlock:
549 State = ControlBlock;
562 if (InStream.SkipBlock())
567 case llvm::BitstreamEntry::EndBlock:
575 unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
580 unsigned Idx = 0, N = Record.size();
591 off_t StoredSize = (off_t)Record[Idx++];
592 time_t StoredModTime = (time_t)Record[Idx++];
599 unsigned Length = Record[Idx++];
601 Record.begin() + Idx +
Length);
606 = FileMgr.getFile(ImportedFile,
false,
608 if (!DependsOnFile ||
609 (StoredSize != DependsOnFile->
getSize()) ||
614 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
615 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
624 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
625 std::unique_ptr<InterestingIdentifierTable> Table(
627 (
const unsigned char *)Blob.data() + Record[0],
628 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
629 (
const unsigned char *)Blob.data()));
630 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
631 DEnd = Table->data_end();
633 std::pair<StringRef, bool> Ident = *D;
635 InterestingIdentifiers[Ident.first].push_back(ID);
637 (
void)InterestingIdentifiers[Ident.first];
651 class IdentifierIndexWriterTrait {
653 typedef StringRef key_type;
654 typedef StringRef key_type_ref;
657 typedef unsigned hash_value_type;
658 typedef unsigned offset_type;
660 static hash_value_type
ComputeHash(key_type_ref Key) {
661 return llvm::HashString(Key);
664 std::pair<unsigned,unsigned>
665 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
666 using namespace llvm::support;
667 endian::Writer<little> LE(Out);
668 unsigned KeyLen = Key.size();
669 unsigned DataLen = Data.size() * 4;
670 LE.write<uint16_t>(KeyLen);
671 LE.write<uint16_t>(DataLen);
672 return std::make_pair(KeyLen, DataLen);
675 void EmitKey(raw_ostream& Out, key_type_ref Key,
unsigned KeyLen) {
676 Out.write(Key.data(), KeyLen);
679 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
681 using namespace llvm::support;
682 for (
unsigned I = 0, N = Data.size();
I != N; ++
I)
683 endian::Writer<little>(Out).write<uint32_t>(Data[
I]);
689 void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
690 using namespace llvm;
693 Stream.Emit((
unsigned)
'B', 8);
694 Stream.Emit((
unsigned)
'C', 8);
695 Stream.Emit((
unsigned)
'G', 8);
696 Stream.Emit((
unsigned)
'I', 8);
700 emitBlockInfoBlock(Stream);
702 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
707 Stream.EmitRecord(INDEX_METADATA, Record);
711 MEnd = ModuleFiles.end();
714 Record.push_back(M->second.ID);
715 Record.push_back(M->first->getSize());
716 Record.push_back(M->first->getModificationTime());
719 StringRef
Name(M->first->getName());
720 Record.push_back(
Name.size());
721 Record.append(
Name.begin(),
Name.end());
724 Record.push_back(M->second.Dependencies.size());
725 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
726 Stream.EmitRecord(MODULE, Record);
731 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
732 IdentifierIndexWriterTrait Trait;
736 IEnd = InterestingIdentifiers.end();
738 Generator.insert(
I->first(),
I->second, Trait);
743 uint32_t BucketOffset;
745 using namespace llvm::support;
746 llvm::raw_svector_ostream Out(IdentifierTable);
748 endian::Writer<little>(Out).write<uint32_t>(0);
749 BucketOffset = Generator.Emit(Out, Trait);
753 BitCodeAbbrev *Abbrev =
new BitCodeAbbrev();
754 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
755 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
756 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
757 unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
760 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
761 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
777 llvm::LockFileManager Locked(IndexPath);
779 case llvm::LockFileManager::LFS_Error:
782 case llvm::LockFileManager::LFS_Owned:
786 case llvm::LockFileManager::LFS_Shared:
793 GlobalModuleIndexBuilder
Builder(FileMgr, PCHContainerRdr);
797 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
801 if (llvm::sys::path::extension(D->path()) !=
".pcm") {
805 if (llvm::sys::path::extension(D->path()) ==
".pcm.lock")
817 if (Builder.loadModuleFile(ModuleFile))
824 llvm::BitstreamWriter OutputStream(OutputBuffer);
825 Builder.writeIndex(OutputStream);
831 if (llvm::sys::fs::createUniqueFile(IndexPath +
"-%%%%%%%%", TmpFD,
836 llvm::raw_fd_ostream Out(TmpFD,
true);
841 Out.write(OutputBuffer.data(), OutputBuffer.size());
847 llvm::sys::fs::remove(IndexPath);
850 if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
852 llvm::sys::fs::remove(IndexTmpPath);
863 IdentifierIndexTable::key_iterator
Current;
866 IdentifierIndexTable::key_iterator
End;
869 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
874 StringRef
Next()
override {
886 IdentifierIndexTable &Table =
887 *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
888 return new GlobalIndexIdentifierIterator(Table);
void getKnownModules(SmallVectorImpl< ModuleFile * > &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void getModuleDependencies(ModuleFile *File, SmallVectorImpl< ModuleFile * > &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
static const unsigned CurrentVersion
The global index file version.
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
detail::InMemoryDirectory::const_iterator I
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, StringRef Path)
Write a global index into the given.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
The result type of a method or function.
Record code for the identifier table.
IndexRecordTypes
Describes the record types in the index.
const TemplateArgument * iterator
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
Cached information about one file (either on disk or in the virtual file system). ...
The AST block, which acts as a container around the full AST block.
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Record code for the list of other AST files imported by this AST file.
There was an unspecified I/O error reading or writing the index.
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(StringRef Path)
Read a global index file for the given directory.
time_t getModificationTime() const
BoundNodesTreeBuilder *const Builder
Dump information about a module file.
The control block, which contains all of the information that needs to be validated prior to committi...
bool lookupIdentifier(StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.