Index: include/clang/Lex/PreprocessingRecord.h =================================================================== --- include/clang/Lex/PreprocessingRecord.h +++ include/clang/Lex/PreprocessingRecord.h @@ -281,6 +281,9 @@ FileID FID) { return None; } + + /// \brief Read a preallocated skipped range from the external source. + virtual SourceRange ReadSkippedRange(unsigned Index) = 0; }; /// \brief A record of the steps taken while preprocessing a source file, @@ -306,6 +309,8 @@ /// \brief The set of ranges that were skipped by the preprocessor, std::vector SkippedRanges; + bool SkippedRangesAllLoaded; + /// \brief Global (loaded or local) ID for a preprocessed entity. /// Negative values are used to indicate preprocessed entities /// loaded from the external source while non-negative values are used to @@ -358,6 +363,15 @@ /// corresponds to the first newly-allocated entity. unsigned allocateLoadedEntities(unsigned NumEntities); + /// \brief Allocate space for a new set of loaded preprocessed skipped ranges. + /// + /// \returns The index into the set of loaded preprocessed ranges, which + /// corresponds to the first newly-allocated range. + unsigned allocateSkippedRanges(unsigned NumRanges); + + /// \brief Ensures that all external skipped ranges have been loaded. + void ensureSkippedRangesLoaded(); + /// \brief Register a new macro definition. void RegisterMacroDefinition(MacroInfo *Macro, MacroDefinitionRecord *Def); @@ -480,7 +494,8 @@ MacroDefinitionRecord *findMacroDefinition(const MacroInfo *MI); /// \brief Retrieve all ranges that got skipped while preprocessing. - const std::vector &getSkippedRanges() const { + const std::vector &getSkippedRanges() { + ensureSkippedRangesLoaded(); return SkippedRanges; } Index: include/clang/Serialization/ASTBitCodes.h =================================================================== --- include/clang/Serialization/ASTBitCodes.h +++ include/clang/Serialization/ASTBitCodes.h @@ -198,6 +198,25 @@ } }; + /// \brief Source range of a skipped preprocessor region + struct PPSkippedRange { + /// \brief Raw source location of beginning of range. + unsigned Begin; + /// \brief Raw source location of end of range. + unsigned End; + + PPSkippedRange(SourceRange R) + : Begin(R.getBegin().getRawEncoding()), + End(R.getEnd().getRawEncoding()) { } + + SourceLocation getBegin() const { + return SourceLocation::getFromRawEncoding(Begin); + } + SourceLocation getEnd() const { + return SourceLocation::getFromRawEncoding(End); + } + }; + /// \brief Source range/offset of a preprocessed entity. struct DeclOffset { /// \brief Raw source location. @@ -627,6 +646,9 @@ /// \brief The stack of open #ifs/#ifdefs recorded in a preamble. PP_CONDITIONAL_STACK = 62, + + /// \brief A table of skipped ranges within the preprocessing record. + PPD_SKIPPED_RANGES = 63 }; /// \brief Record types used within a source manager block. Index: include/clang/Serialization/ASTReader.h =================================================================== --- include/clang/Serialization/ASTReader.h +++ include/clang/Serialization/ASTReader.h @@ -753,6 +753,13 @@ /// added to the global preprocessing entity ID to produce a local ID. GlobalPreprocessedEntityMapType GlobalPreprocessedEntityMap; + typedef ContinuousRangeMap + GlobalSkippedRangeMapType; + + /// \brief Mapping from global skipped range base IDs to the module in which + /// the skipped ranges reside. + GlobalSkippedRangeMapType GlobalSkippedRangeMap; + /// \name CodeGen-relevant special data /// \brief Fields containing data that is relevant to CodeGen. //@{ @@ -1689,6 +1696,9 @@ Optional isPreprocessedEntityInFileID(unsigned Index, FileID FID) override; + /// \brief Read a preallocated skipped range from the external source. + SourceRange ReadSkippedRange(unsigned Index) override; + /// \brief Read the header file information for the given file entry. HeaderFileInfo GetHeaderFileInfo(const FileEntry *FE) override; Index: include/clang/Serialization/Module.h =================================================================== --- include/clang/Serialization/Module.h +++ include/clang/Serialization/Module.h @@ -324,6 +324,12 @@ const PPEntityOffset *PreprocessedEntityOffsets = nullptr; unsigned NumPreprocessedEntities = 0; + /// \brief Base ID for preprocessed skipped ranges local to this module. + unsigned BasePreprocessedSkippedRangeID = 0; + + const PPSkippedRange *PreprocessedSkippedRangeOffsets = nullptr; + unsigned NumPreprocessedSkippedRanges = 0; + // === Header search information === /// \brief The number of local HeaderFileInfo structures. Index: lib/Lex/PPDirectives.cpp =================================================================== --- lib/Lex/PPDirectives.cpp +++ lib/Lex/PPDirectives.cpp @@ -558,7 +558,9 @@ // the #if block. CurPPLexer->LexingRawMode = false; - if (Callbacks) + // The last skipped range isn't actually skipped yet if it's truncated + // by the end of the preamble; we'll resume parsing after the preamble. + if (Callbacks && (Tok.isNot(tok::eof) || !isRecordingPreamble())) Callbacks->SourceRangeSkipped( SourceRange(HashTokenLoc, CurPPLexer->getSourceLocation()), Tok.getLocation()); Index: lib/Lex/PreprocessingRecord.cpp =================================================================== --- lib/Lex/PreprocessingRecord.cpp +++ lib/Lex/PreprocessingRecord.cpp @@ -35,7 +35,8 @@ PreprocessingRecord::PreprocessingRecord(SourceManager &SM) : SourceMgr(SM), - ExternalSource(nullptr) { + ExternalSource(nullptr), + SkippedRangesAllLoaded(true) { } /// \brief Returns a pair of [Begin, End) iterators of preprocessed entities @@ -313,6 +314,24 @@ return Result; } +unsigned PreprocessingRecord::allocateSkippedRanges(unsigned NumRanges) { + unsigned Result = SkippedRanges.size(); + SkippedRanges.resize(SkippedRanges.size() + NumRanges); + SkippedRangesAllLoaded = false; + return Result; +} + +void PreprocessingRecord::ensureSkippedRangesLoaded() { + if (SkippedRangesAllLoaded || !ExternalSource) + return; + for (unsigned Index = 0; Index != SkippedRanges.size(); ++Index) { + if (SkippedRanges[Index].isInvalid()) + SkippedRanges[Index] = ExternalSource->ReadSkippedRange(Index); + } + SkippedRangesAllLoaded = true; +} + + void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro, MacroDefinitionRecord *Def) { MacroDefinitions[Macro] = Def; @@ -402,6 +421,7 @@ void PreprocessingRecord::SourceRangeSkipped(SourceRange Range, SourceLocation EndifLoc) { + assert(Range.isValid()); SkippedRanges.emplace_back(Range.getBegin(), EndifLoc); } @@ -481,5 +501,6 @@ return BumpAlloc.getTotalMemory() + llvm::capacity_in_bytes(MacroDefinitions) + llvm::capacity_in_bytes(PreprocessedEntities) - + llvm::capacity_in_bytes(LoadedPreprocessedEntities); + + llvm::capacity_in_bytes(LoadedPreprocessedEntities) + + llvm::capacity_in_bytes(SkippedRanges); } Index: lib/Serialization/ASTReader.cpp =================================================================== --- lib/Serialization/ASTReader.cpp +++ lib/Serialization/ASTReader.cpp @@ -3214,6 +3214,24 @@ break; } + case PPD_SKIPPED_RANGES: { + F.PreprocessedSkippedRangeOffsets = (const PPSkippedRange*)Blob.data(); + assert(Blob.size() % sizeof(PPSkippedRange) == 0); + F.NumPreprocessedSkippedRanges = Blob.size() / sizeof(PPSkippedRange); + + if (!PP.getPreprocessingRecord()) + PP.createPreprocessingRecord(); + if (!PP.getPreprocessingRecord()->getExternalSource()) + PP.getPreprocessingRecord()->SetExternalSource(*this); + F.BasePreprocessedSkippedRangeID = PP.getPreprocessingRecord() + ->allocateSkippedRanges(F.NumPreprocessedSkippedRanges); + + if (F.NumPreprocessedSkippedRanges > 0) + GlobalSkippedRangeMap.insert( + std::make_pair(F.BasePreprocessedSkippedRangeID, &F)); + break; + } + case DECL_UPDATE_OFFSETS: if (Record.size() % 2 != 0) { Error("invalid DECL_UPDATE_OFFSETS block in AST file"); @@ -5379,6 +5397,20 @@ Mod.FileSortedDecls + Mod.NumFileSortedDecls)); } +SourceRange ASTReader::ReadSkippedRange(unsigned GlobalIndex) { + auto I = GlobalSkippedRangeMap.find(GlobalIndex); + assert(I != GlobalSkippedRangeMap.end() && + "Corrupted global skipped range map"); + ModuleFile *M = I->second; + unsigned LocalIndex = GlobalIndex - M->BasePreprocessedSkippedRangeID; + assert(LocalIndex < M->NumPreprocessedSkippedRanges); + PPSkippedRange RawRange = M->PreprocessedSkippedRangeOffsets[LocalIndex]; + SourceRange Range(TranslateSourceLocation(*M, RawRange.getBegin()), + TranslateSourceLocation(*M, RawRange.getEnd())); + assert(Range.isValid()); + return Range; +} + PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) { PreprocessedEntityID PPID = Index+1; std::pair PPInfo = getModulePreprocessedEntity(Index); Index: lib/Serialization/ASTWriter.cpp =================================================================== --- lib/Serialization/ASTWriter.cpp +++ lib/Serialization/ASTWriter.cpp @@ -1104,6 +1104,7 @@ RECORD(UNUSED_FILESCOPED_DECLS); RECORD(PPD_ENTITIES_OFFSETS); RECORD(VTABLE_USES); + RECORD(PPD_SKIPPED_RANGES); RECORD(REFERENCED_SELECTOR_POOL); RECORD(TU_UPDATE_LEXICAL); RECORD(SEMA_DECL_REFS); @@ -2725,6 +2726,26 @@ Stream.EmitRecordWithBlob(PPEOffsetAbbrev, Record, bytes(PreprocessedEntityOffsets)); } + + // Write the skipped region table for the preprocessing record. + ArrayRef SkippedRanges = PPRec.getSkippedRanges(); + if (SkippedRanges.size() > 0) { + std::vector SerializedSkippedRanges; + SerializedSkippedRanges.reserve(SkippedRanges.size()); + for (auto const& Range : SkippedRanges) + SerializedSkippedRanges.emplace_back(Range); + + using namespace llvm; + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(PPD_SKIPPED_RANGES)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned PPESkippedRangeAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); + + Record.clear(); + Record.push_back(PPD_SKIPPED_RANGES); + Stream.EmitRecordWithBlob(PPESkippedRangeAbbrev, Record, + bytes(SerializedSkippedRanges)); + } } unsigned ASTWriter::getLocalOrImportedSubmoduleID(Module *Mod) { Index: unittests/libclang/LibclangTest.cpp =================================================================== --- unittests/libclang/LibclangTest.cpp +++ unittests/libclang/LibclangTest.cpp @@ -572,3 +572,52 @@ EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); DisplayDiagnostics(); } + +TEST_F(LibclangReparseTest, PreprocessorSkippedRanges) { + std::string Header = "header.h", Main = "main.cpp"; + WriteFile(Header, + "#ifdef MANGOS\n" + "printf(\"mmm\");\n" + "#endif"); + WriteFile(Main, + "#include \"header.h\"\n" + "#ifdef KIWIS\n" + "printf(\"mmm!!\");\n" + "#endif"); + + unsigned flags = TUFlags | + CXTranslationUnit_PrecompiledPreamble;//| + //CXTranslationUnit_CreatePreambleOnFirstParse; + for (int i = 0; i != 5; ++i) { + if (i == 0) { + // first parse + ClangTU = clang_parseTranslationUnit(Index, Main.c_str(), nullptr, 0, + nullptr, 0, flags); + } + else { + // reparse + ASSERT_TRUE(ReparseTU(0, nullptr /* No unsaved files. */)); + } + + CXSourceRangeList *Ranges = clang_getAllSkippedRanges(ClangTU); + EXPECT_EQ(2U, Ranges->count); + + CXSourceLocation cxl; + unsigned line; + cxl = clang_getRangeStart(Ranges->ranges[0]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(1U, line); + cxl = clang_getRangeEnd(Ranges->ranges[0]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(3U, line); + + cxl = clang_getRangeStart(Ranges->ranges[1]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(2U, line); + cxl = clang_getRangeEnd(Ranges->ranges[1]); + clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr); + EXPECT_EQ(4U, line); + + clang_disposeSourceRangeList(Ranges); + } +}