Index: include/clang/Lex/PreprocessingRecord.h =================================================================== --- include/clang/Lex/PreprocessingRecord.h +++ include/clang/Lex/PreprocessingRecord.h @@ -297,6 +297,9 @@ FileID FID) { return None; } + + /// \brief Read a preallocated skipped range from the external source. + virtual SourceRange ReadSkippedRange(unsigned Index) = 0; }; /// \brief A record of the steps taken while preprocessing a source file, @@ -322,6 +325,8 @@ /// \brief The set of ranges that were skipped by the preprocessor, std::vector SkippedRanges; + bool SkippedRangesAllLoaded = true; + /// \brief Global (loaded or local) ID for a preprocessed entity. /// Negative values are used to indicate preprocessed entities /// loaded from the external source while non-negative values are used to @@ -377,6 +382,16 @@ /// corresponds to the first newly-allocated entity. unsigned allocateLoadedEntities(unsigned NumEntities); + /// \brief Allocate space for a new set of loaded preprocessed skipped + /// ranges. + /// + /// \returns The index into the set of loaded preprocessed ranges, which + /// corresponds to the first newly-allocated range. + unsigned allocateSkippedRanges(unsigned NumRanges); + + /// \brief Ensures that all external skipped ranges have been loaded. + void ensureSkippedRangesLoaded(); + /// \brief Register a new macro definition. void RegisterMacroDefinition(MacroInfo *Macro, MacroDefinitionRecord *Def); @@ -499,7 +514,8 @@ MacroDefinitionRecord *findMacroDefinition(const MacroInfo *MI); /// \brief Retrieve all ranges that got skipped while preprocessing. - const std::vector &getSkippedRanges() const { + const std::vector &getSkippedRanges() { + ensureSkippedRangesLoaded(); return SkippedRanges; } Index: include/clang/Serialization/ASTBitCodes.h =================================================================== --- include/clang/Serialization/ASTBitCodes.h +++ include/clang/Serialization/ASTBitCodes.h @@ -198,6 +198,25 @@ } }; + /// \brief Source range of a skipped preprocessor region + struct PPSkippedRange { + /// \brief Raw source location of beginning of range. + unsigned Begin; + /// \brief Raw source location of end of range. + unsigned End; + + PPSkippedRange(SourceRange R) + : Begin(R.getBegin().getRawEncoding()), + End(R.getEnd().getRawEncoding()) { } + + SourceLocation getBegin() const { + return SourceLocation::getFromRawEncoding(Begin); + } + SourceLocation getEnd() const { + return SourceLocation::getFromRawEncoding(End); + } + }; + /// \brief Source range/offset of a preprocessed entity. struct DeclOffset { /// \brief Raw source location. @@ -627,6 +646,9 @@ /// \brief The stack of open #ifs/#ifdefs recorded in a preamble. PP_CONDITIONAL_STACK = 62, + + /// \brief A table of skipped ranges within the preprocessing record. + PPD_SKIPPED_RANGES = 63 }; /// \brief Record types used within a source manager block. Index: include/clang/Serialization/ASTReader.h =================================================================== --- include/clang/Serialization/ASTReader.h +++ include/clang/Serialization/ASTReader.h @@ -753,6 +753,13 @@ /// added to the global preprocessing entity ID to produce a local ID. GlobalPreprocessedEntityMapType GlobalPreprocessedEntityMap; + typedef ContinuousRangeMap + GlobalSkippedRangeMapType; + + /// \brief Mapping from global skipped range base IDs to the module in which + /// the skipped ranges reside. + GlobalSkippedRangeMapType GlobalSkippedRangeMap; + /// \name CodeGen-relevant special data /// \brief Fields containing data that is relevant to CodeGen. //@{ @@ -1693,6 +1700,9 @@ Optional isPreprocessedEntityInFileID(unsigned Index, FileID FID) override; + /// \brief Read a preallocated skipped range from the external source. + SourceRange ReadSkippedRange(unsigned Index) override; + /// \brief Read the header file information for the given file entry. HeaderFileInfo GetHeaderFileInfo(const FileEntry *FE) override; Index: include/clang/Serialization/Module.h =================================================================== --- include/clang/Serialization/Module.h +++ include/clang/Serialization/Module.h @@ -324,6 +324,12 @@ const PPEntityOffset *PreprocessedEntityOffsets = nullptr; unsigned NumPreprocessedEntities = 0; + /// \brief Base ID for preprocessed skipped ranges local to this module. + unsigned BasePreprocessedSkippedRangeID = 0; + + const PPSkippedRange *PreprocessedSkippedRangeOffsets = nullptr; + unsigned NumPreprocessedSkippedRanges = 0; + // === Header search information === /// \brief The number of local HeaderFileInfo structures. Index: lib/Lex/PPDirectives.cpp =================================================================== --- lib/Lex/PPDirectives.cpp +++ lib/Lex/PPDirectives.cpp @@ -558,7 +558,9 @@ // the #if block. CurPPLexer->LexingRawMode = false; - if (Callbacks) + // The last skipped range isn't actually skipped yet if it's truncated + // by the end of the preamble; we'll resume parsing after the preamble. + if (Callbacks && (Tok.isNot(tok::eof) || !isRecordingPreamble())) Callbacks->SourceRangeSkipped( SourceRange(HashTokenLoc, CurPPLexer->getSourceLocation()), Tok.getLocation()); Index: lib/Lex/PreprocessingRecord.cpp =================================================================== --- lib/Lex/PreprocessingRecord.cpp +++ lib/Lex/PreprocessingRecord.cpp @@ -329,6 +329,23 @@ return Result; } +unsigned PreprocessingRecord::allocateSkippedRanges(unsigned NumRanges) { + unsigned Result = SkippedRanges.size(); + SkippedRanges.resize(SkippedRanges.size() + NumRanges); + SkippedRangesAllLoaded = false; + return Result; +} + +void PreprocessingRecord::ensureSkippedRangesLoaded() { + if (SkippedRangesAllLoaded || !ExternalSource) + return; + for (unsigned Index = 0; Index != SkippedRanges.size(); ++Index) { + if (SkippedRanges[Index].isInvalid()) + SkippedRanges[Index] = ExternalSource->ReadSkippedRange(Index); + } + SkippedRangesAllLoaded = true; +} + void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro, MacroDefinitionRecord *Def) { MacroDefinitions[Macro] = Def; @@ -418,6 +435,7 @@ void PreprocessingRecord::SourceRangeSkipped(SourceRange Range, SourceLocation EndifLoc) { + assert(Range.isValid()); SkippedRanges.emplace_back(Range.getBegin(), EndifLoc); } @@ -497,5 +515,6 @@ return BumpAlloc.getTotalMemory() + llvm::capacity_in_bytes(MacroDefinitions) + llvm::capacity_in_bytes(PreprocessedEntities) - + llvm::capacity_in_bytes(LoadedPreprocessedEntities); + + llvm::capacity_in_bytes(LoadedPreprocessedEntities) + + llvm::capacity_in_bytes(SkippedRanges); } Index: lib/Serialization/ASTReader.cpp =================================================================== --- lib/Serialization/ASTReader.cpp +++ lib/Serialization/ASTReader.cpp @@ -3215,6 +3215,24 @@ break; } + case PPD_SKIPPED_RANGES: { + F.PreprocessedSkippedRangeOffsets = (const PPSkippedRange*)Blob.data(); + assert(Blob.size() % sizeof(PPSkippedRange) == 0); + F.NumPreprocessedSkippedRanges = Blob.size() / sizeof(PPSkippedRange); + + if (!PP.getPreprocessingRecord()) + PP.createPreprocessingRecord(); + if (!PP.getPreprocessingRecord()->getExternalSource()) + PP.getPreprocessingRecord()->SetExternalSource(*this); + F.BasePreprocessedSkippedRangeID = PP.getPreprocessingRecord() + ->allocateSkippedRanges(F.NumPreprocessedSkippedRanges); + + if (F.NumPreprocessedSkippedRanges > 0) + GlobalSkippedRangeMap.insert( + std::make_pair(F.BasePreprocessedSkippedRangeID, &F)); + break; + } + case DECL_UPDATE_OFFSETS: if (Record.size() % 2 != 0) { Error("invalid DECL_UPDATE_OFFSETS block in AST file"); @@ -5387,6 +5405,20 @@ Mod.FileSortedDecls + Mod.NumFileSortedDecls)); } +SourceRange ASTReader::ReadSkippedRange(unsigned GlobalIndex) { + auto I = GlobalSkippedRangeMap.find(GlobalIndex); + assert(I != GlobalSkippedRangeMap.end() && + "Corrupted global skipped range map"); + ModuleFile *M = I->second; + unsigned LocalIndex = GlobalIndex - M->BasePreprocessedSkippedRangeID; + assert(LocalIndex < M->NumPreprocessedSkippedRanges); + PPSkippedRange RawRange = M->PreprocessedSkippedRangeOffsets[LocalIndex]; + SourceRange Range(TranslateSourceLocation(*M, RawRange.getBegin()), + TranslateSourceLocation(*M, RawRange.getEnd())); + assert(Range.isValid()); + return Range; +} + PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) { PreprocessedEntityID PPID = Index+1; std::pair PPInfo = getModulePreprocessedEntity(Index); Index: lib/Serialization/ASTWriter.cpp =================================================================== --- lib/Serialization/ASTWriter.cpp +++ lib/Serialization/ASTWriter.cpp @@ -1104,6 +1104,7 @@ RECORD(UNUSED_FILESCOPED_DECLS); RECORD(PPD_ENTITIES_OFFSETS); RECORD(VTABLE_USES); + RECORD(PPD_SKIPPED_RANGES); RECORD(REFERENCED_SELECTOR_POOL); RECORD(TU_UPDATE_LEXICAL); RECORD(SEMA_DECL_REFS); @@ -2726,6 +2727,26 @@ Stream.EmitRecordWithBlob(PPEOffsetAbbrev, Record, bytes(PreprocessedEntityOffsets)); } + + // Write the skipped region table for the preprocessing record. + ArrayRef SkippedRanges = PPRec.getSkippedRanges(); + if (SkippedRanges.size() > 0) { + std::vector SerializedSkippedRanges; + SerializedSkippedRanges.reserve(SkippedRanges.size()); + for (auto const& Range : SkippedRanges) + SerializedSkippedRanges.emplace_back(Range); + + using namespace llvm; + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(PPD_SKIPPED_RANGES)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned PPESkippedRangeAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); + + Record.clear(); + Record.push_back(PPD_SKIPPED_RANGES); + Stream.EmitRecordWithBlob(PPESkippedRangeAbbrev, Record, + bytes(SerializedSkippedRanges)); + } } unsigned ASTWriter::getLocalOrImportedSubmoduleID(Module *Mod) { Index: tools/libclang/CIndex.cpp =================================================================== --- tools/libclang/CIndex.cpp +++ tools/libclang/CIndex.cpp @@ -8160,6 +8160,7 @@ SourceManager &sm = Ctx.getSourceManager(); FileEntry *fileEntry = static_cast(file); FileID wantedFileID = sm.translateFile(fileEntry); + bool isMainFile = wantedFileID == sm.getMainFileID(); const std::vector &SkippedRanges = ppRec->getSkippedRanges(); std::vector wantedRanges; @@ -8167,6 +8168,8 @@ i != ei; ++i) { if (sm.getFileID(i->getBegin()) == wantedFileID || sm.getFileID(i->getEnd()) == wantedFileID) wantedRanges.push_back(*i); + else if (isMainFile && (astUnit->isInPreambleFileID(i->getBegin()) || astUnit->isInPreambleFileID(i->getEnd()))) + wantedRanges.push_back(*i); } skipped->count = wantedRanges.size(); Index: unittests/libclang/LibclangTest.cpp =================================================================== --- unittests/libclang/LibclangTest.cpp +++ unittests/libclang/LibclangTest.cpp @@ -572,3 +572,81 @@ EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); DisplayDiagnostics(); } + +TEST_F(LibclangReparseTest, PreprocessorSkippedRanges) { + std::string Header = "header.h", Main = "main.cpp"; + WriteFile(Header, + "#ifdef MANGOS\n" + "printf(\"mmm\");\n" + "#endif"); + WriteFile(Main, + "#include \"header.h\"\n" + "#ifdef GUAVA\n" + "#endif\n" + "#ifdef KIWIS\n" + "printf(\"mmm!!\");\n" + "#endif"); + + for (int i = 0; i != 3; ++i) { + unsigned flags = TUFlags | CXTranslationUnit_PrecompiledPreamble; + if (i == 2) + flags |= CXTranslationUnit_CreatePreambleOnFirstParse; + + // parse once + ClangTU = clang_parseTranslationUnit(Index, Main.c_str(), nullptr, 0, + nullptr, 0, flags); + if (i != 0) { + // reparse + ASSERT_TRUE(ReparseTU(0, nullptr /* No unsaved files. */)); + } + + // Check all ranges are there + CXSourceRangeList *Ranges = clang_getAllSkippedRanges(ClangTU); + EXPECT_EQ(3U, Ranges->count); + + CXSourceLocation cxl; + unsigned line; + cxl = clang_getRangeStart(Ranges->ranges[0]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(1U, line); + cxl = clang_getRangeEnd(Ranges->ranges[0]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(3U, line); + + cxl = clang_getRangeStart(Ranges->ranges[1]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(2U, line); + cxl = clang_getRangeEnd(Ranges->ranges[1]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(3U, line); + + cxl = clang_getRangeStart(Ranges->ranges[2]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(4U, line); + cxl = clang_getRangeEnd(Ranges->ranges[2]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(6U, line); + + clang_disposeSourceRangeList(Ranges); + + // Check obtaining ranges by each file works + CXFile cxf = clang_getFile(ClangTU, Header.c_str()); + Ranges = clang_getSkippedRanges(ClangTU, cxf); + EXPECT_EQ(1U, Ranges->count); + cxl = clang_getRangeStart(Ranges->ranges[0]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(1U, line); + clang_disposeSourceRangeList(Ranges); + + cxf = clang_getFile(ClangTU, Main.c_str()); + Ranges = clang_getSkippedRanges(ClangTU, cxf); + EXPECT_EQ(2U, Ranges->count); + cxl = clang_getRangeStart(Ranges->ranges[0]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(2U, line); + cxl = clang_getRangeStart(Ranges->ranges[1]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(4U, line); + clang_disposeSourceRangeList(Ranges); + } +}